Skip to content

Automate browser based workflows with AI

Send the home prompt on Enter, Shift+Enter for a new line (#8741)

Cindy Li3820f6637733
diff --git a/docs/changelog.mdx b/docs/changelog.mdx
index 0e665635c..5de0ca2dc 100644
--- a/docs/changelog.mdx
+++ b/docs/changelog.mdx
@@ -59,6 +59,8 @@ keywords:
- **Downloadable Copilot-Generated Files** — Secure CodeBlocks and Copilot can now publish generated XLSX, PDF, CSV, and ZIP files as run downloads and chat download buttons. ([#8693](https://github.com/Skyvern-AI/skyvern/pull/8693))
- **Clipboard Paste in Live Browser Control** — You can now paste your clipboard while in control of a live browser, from one control bar shared by CDP and VNC sessions. ([#8699](https://github.com/Skyvern-AI/skyvern/pull/8699))
- **Redesigned Discover Home** — The Discover page's prompt, examples, and templates have been redesigned.
+- **Guided Full-Page Welcome for New Sign-Ups** — New accounts now land on a full-page walkthrough covering goals, expected volume, and role, replacing the old setup modal.
+- **Guided Template Walkthrough** — Using a home page template now copies it, opens Copilot with a numbered summary of what it does, and shows an inputs card for the credentials and values it needs. ([#8730](https://github.com/Skyvern-AI/skyvern/pull/8730))
## Improvements
@@ -79,6 +81,13 @@ keywords:
- **Clearer Billing Errors** — Skyvern now shows the reason when a billing plan change fails.
- **Faster Credential Failures** — A login block now fails fast with a clear reason when it isn't bound to any of the workflow's credentials, instead of opening a browser first.
- **Copilot Test Results as Cards** — Each block Copilot tests now shows as one card with its status, code change, and failing line.
+- **Copilot Run History by Status** — Copilot can now list a workflow's recent runs filtered by status, such as failed or terminated, instead of only seeing the most recent run.
+- **Longer Task Runs on Long Forms** — Task runs now get more time to finish long forms that are still making visible progress, instead of stopping once the token budget runs out.
+- **Unsubscribe from Skyvern Emails** — You can now unsubscribe from categories of email Skyvern sends, like the analytics digest, with a one-click link.
+- **Default Engine Option in Workflow Editor** — The block engine picker now has a "Default" option separate from Skyvern 1.0 and 2.0, which are now labeled Legacy. ([#8735](https://github.com/Skyvern-AI/skyvern/pull/8735))
+- **Docked Copilot Sign-In Requests** — Copilot's sign-in and Google account requests now dock above the chat input instead of scrolling out of view with the transcript.
+- **More Reliable Copilot Repairs on SSL Errors** — Copilot no longer gives up early on ambiguous SSL connection errors during automatic repairs; it now attempts a fix. ([#8731](https://github.com/Skyvern-AI/skyvern/pull/8731))
+- **Upstream Output Reuse in Copilot Repair Tests** — Testing a repaired block that reads an earlier block's output no longer fails with a template error; the test now reuses that output automatically. ([#8737](https://github.com/Skyvern-AI/skyvern/pull/8737))
## Bug fixes
@@ -114,6 +123,12 @@ keywords:
- Fixed one-shot downloads not recovering correctly after a server error.
- Fixed the organizations page showing a blank state instead of a full-page retry option when it failed to load.
- Fixed Workflow Studio showing replay pills before there was anything to replay.
+- Fixed a Cloudflare Turnstile checkbox being reported as solved after it silently reset, instead of being re-clicked.
+- Fixed browser pages hanging for other users sharing a browser session after one person disconnected mid-interaction. ([#8736](https://github.com/Skyvern-AI/skyvern/pull/8736))
+- Fixed Copilot failing before sending any request when using certain GPT-6 models.
+- Fixed task runs refusing to click a checkbox that sits on top of its own styled replacement, reporting it as not rendered. ([#8734](https://github.com/Skyvern-AI/skyvern/pull/8734))
+- Fixed task runs losing track of a page after a click opened a new tab, causing the run to stall. ([#8733](https://github.com/Skyvern-AI/skyvern/pull/8733))
+- Fixed a dark-mode flash of a white screen on sign-in, refresh, and other full page loads.
</Update>
diff --git a/skyvern-frontend/src/components/BrowserStream.test.tsx b/skyvern-frontend/src/components/BrowserStream.test.tsx
index 190d52d8a..2eb273a06 100644
--- a/skyvern-frontend/src/components/BrowserStream.test.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.test.tsx
@@ -1,12 +1,14 @@
import { StrictMode, type ComponentProps } from "react";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import {
+ act,
cleanup,
fireEvent,
render,
screen,
waitFor,
} from "@testing-library/react";
+import { AxiosError, type AxiosResponse } from "axios";
import { MemoryRouter, Route, Routes } from "react-router-dom";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
@@ -218,10 +220,46 @@ vi.mock("@/store/useRecordingStore", () => {
};
});
+const respondWithRunningSession = mocks.apiGet.getMockImplementation()!;
+
+type SessionFields = {
+ status: string;
+ browser_address: string | null;
+ started_at: string | null;
+ completed_at: string | null;
+};
+
+async function sessionWith(fields: Partial<SessionFields>) {
+ return { ...(await respondWithRunningSession()).data, ...fields };
+}
+
+function respondWith(respond: () => Promise<{ data: unknown }>) {
+ mocks.apiGet.mockImplementation(respond as typeof respondWithRunningSession);
+}
+
+function failWith(status: number) {
+ respondWith(async () => {
+ throw new AxiosError("request failed", undefined, undefined, undefined, {
+ status,
+ } as AxiosResponse);
+ });
+}
+
+// Through act, so React settles the renders and effects each tick schedules.
+function advance(ms: number) {
+ return act(() => vi.advanceTimersByTimeAsync(ms));
+}
+
+function sessionRequestCount() {
+ return (mocks.apiGet.mock.calls as unknown[][]).filter(
+ ([url]) => url === "/browser_sessions/pbs_test",
+ ).length;
+}
+
function renderBrowserStream(
props: Pick<
ComponentProps<typeof BrowserStream>,
- "onActivity" | "onStreamStateChange"
+ "browserSession" | "onActivity" | "onStreamStateChange"
> = {},
) {
const queryClient = new QueryClient({
@@ -236,6 +274,7 @@ function renderBrowserStream(
<QueryClientProvider client={queryClient}>
<BrowserStream
browserSessionId="pbs_test"
+ browserSession={props.browserSession}
interactive={false}
showControlButtons={true}
onActivity={props.onActivity}
@@ -307,6 +346,7 @@ describe("BrowserStream", () => {
cleanup();
vi.restoreAllMocks();
vi.clearAllMocks();
+ mocks.apiGet.mockImplementation(respondWithRunningSession);
mocks.rfbInstances.length = 0;
mocks.wsInstances.length = 0;
});
@@ -916,6 +956,194 @@ describe("BrowserStream", () => {
});
});
+ describe("session polling", () => {
+ const WINDOW_MS = 30_000;
+ const STARTED = {};
+ const NOT_STARTED = {
+ status: "created",
+ browser_address: null,
+ started_at: null,
+ };
+
+ beforeEach(() => {
+ vi.useFakeTimers();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it.each([
+ {
+ mount: "the session page",
+ render: renderBrowserSession,
+ phase: "a started",
+ fields: STARTED,
+ intervalMs: 5000,
+ },
+ {
+ mount: "the session page",
+ render: renderBrowserSession,
+ phase: "a not-yet-started",
+ fields: NOT_STARTED,
+ intervalMs: 1000,
+ },
+ {
+ mount: "a mount with no parent session",
+ render: () => renderBrowserStream(),
+ phase: "a started",
+ fields: STARTED,
+ intervalMs: 5000,
+ },
+ {
+ mount: "a mount with no parent session",
+ render: () => renderBrowserStream(),
+ phase: "a not-yet-started",
+ fields: NOT_STARTED,
+ intervalMs: 1000,
+ },
+ ])(
+ "requests $phase session once per interval from $mount",
+ async ({ render: renderMount, fields, intervalMs }) => {
+ const session = await sessionWith(fields);
+ respondWith(async () => ({ data: session }));
+ renderMount();
+ await advance(500);
+ mocks.apiGet.mockClear();
+
+ await advance(WINDOW_MS);
+
+ expect(sessionRequestCount()).toBe(WINDOW_MS / intervalMs);
+ },
+ );
+
+ it("dials VNC from the session page once the page's poll reports the session started", async () => {
+ const notStarted = await sessionWith(NOT_STARTED);
+ respondWith(async () => ({ data: notStarted }));
+ renderBrowserSession();
+ await advance(3000);
+ expect(screen.getByText(/warming up your browser/i)).toBeTruthy();
+ expect(mocks.rfbInstances).toHaveLength(0);
+
+ mocks.apiGet.mockImplementation(respondWithRunningSession);
+ await advance(1500);
+
+ expect(mocks.rfbInstances).toHaveLength(1);
+ });
+
+ describe.each(["its own poll", "the parent's session"] as const)(
+ "reading the session from %s",
+ (source) => {
+ it.each([
+ {
+ state: "completed",
+ fields: { completed_at: "2026-01-01T01:00:00Z" },
+ panel: /wandered off/i,
+ vncDials: 0,
+ streamState: "stopped",
+ },
+ {
+ state: "started, by started_at alone",
+ fields: { browser_address: null },
+ panel: null,
+ vncDials: 1,
+ streamState: "live",
+ },
+ {
+ state: "started, by browser_address alone",
+ fields: { started_at: null },
+ panel: null,
+ vncDials: 1,
+ streamState: "live",
+ },
+ {
+ state: "not yet started",
+ fields: NOT_STARTED,
+ panel: /warming up your browser/i,
+ vncDials: 0,
+ streamState: "connecting",
+ },
+ ])(
+ "a $state session",
+ async ({ fields, panel, vncDials, streamState }) => {
+ const session = await sessionWith(fields);
+ const onStreamStateChange = vi.fn();
+ if (source === "its own poll") {
+ respondWith(async () => ({ data: session }));
+ renderBrowserStream({ onStreamStateChange });
+ } else {
+ renderBrowserStream({
+ onStreamStateChange,
+ browserSession: session,
+ });
+ }
+
+ await advance(1000);
+
+ expect(mocks.rfbInstances).toHaveLength(vncDials);
+ expect(onStreamStateChange).toHaveBeenLastCalledWith(
+ streamState,
+ "pbs_test",
+ );
+ if (panel) {
+ expect(screen.getByText(panel)).toBeTruthy();
+ }
+ expect(sessionRequestCount() > 0).toBe(source === "its own poll");
+ },
+ );
+ },
+ );
+
+ it("treats a missing session as ended", async () => {
+ respondWith(async () => ({ data: null }));
+ const onStreamStateChange = vi.fn();
+ renderBrowserStream({ onStreamStateChange });
+
+ await advance(1000);
+
+ expect(screen.getByText(/wandered off/i)).toBeTruthy();
+ expect(mocks.rfbInstances).toHaveLength(0);
+ expect(onStreamStateChange).toHaveBeenLastCalledWith(
+ "stopped",
+ "pbs_test",
+ );
+ });
+
+ it("keeps polling through a failed request without calling the session ended", async () => {
+ failWith(500);
+ const onStreamStateChange = vi.fn();
+ renderBrowserStream({ onStreamStateChange });
+ await advance(500);
+ mocks.apiGet.mockClear();
+
+ await advance(WINDOW_MS);
+
+ expect(sessionRequestCount()).toBe(WINDOW_MS / 1000);
+ expect(screen.getByText(/wandered off/i)).toBeTruthy();
+ expect(mocks.rfbInstances).toHaveLength(0);
+ expect(onStreamStateChange).toHaveBeenLastCalledWith(
+ "connecting",
+ "pbs_test",
+ );
+ });
+
+ it("stops polling a forbidden session and reports it ended", async () => {
+ failWith(403);
+ const onStreamStateChange = vi.fn();
+ renderBrowserStream({ onStreamStateChange });
+
+ await advance(WINDOW_MS);
+
+ expect(sessionRequestCount()).toBe(1);
+ expect(screen.getByText(/wandered off/i)).toBeTruthy();
+ expect(mocks.rfbInstances).toHaveLength(0);
+ expect(onStreamStateChange).toHaveBeenLastCalledWith(
+ "stopped",
+ "pbs_test",
+ );
+ });
+ });
+
describe("recording reset lifecycle", () => {
it("resets the recording store on unmount by default", async () => {
const { unmount } = renderWithRecordingReset(undefined);
diff --git a/skyvern-frontend/src/components/BrowserStream.tsx b/skyvern-frontend/src/components/BrowserStream.tsx
index 90c5983d1..10e7332f8 100644
--- a/skyvern-frontend/src/components/BrowserStream.tsx
+++ b/skyvern-frontend/src/components/BrowserStream.tsx
@@ -77,8 +77,29 @@ interface BrowserSession {
completed_at?: string | null;
}
+type BrowserSessionLiveness = Pick<
+ BrowserSession,
+ "browser_address" | "started_at" | "completed_at"
+>;
+
+function getBrowserSessionState(browserSession: BrowserSessionLiveness | null) {
+ const hasBrowserSession = Boolean(
+ browserSession && !browserSession.completed_at,
+ );
+ return {
+ hasBrowserSession,
+ isBrowserSessionStarted:
+ hasBrowserSession &&
+ Boolean(browserSession?.started_at || browserSession?.browser_address),
+ isBrowserSessionEnded: !hasBrowserSession,
+ };
+}
+
type Props = {
browserSessionId?: string;
+ // The session behind browserSessionId, from a parent that already polls it.
+ // The stream then reads this instead of polling the session a second time.
+ browserSession?: BrowserSessionLiveness;
exfiltrate?: boolean;
interactive?: boolean;
showControlButtons?: boolean;
@@ -125,6 +146,7 @@ function applyVncStreamProfile(
function BrowserStream({
browserSessionId = undefined,
+ browserSession: parentBrowserSession = undefined,
exfiltrate = false,
interactive = true,
showControlButtons = undefined,
@@ -163,6 +185,10 @@ function BrowserStream({
runId = null;
}
+ const parentSessionState = parentBrowserSession
+ ? getBrowserSessionState(parentBrowserSession)
+ : null;
+
useQuery({
queryKey: ["hasBrowserSession", browserSessionId],
queryFn: async () => {
@@ -172,21 +198,12 @@ function BrowserStream({
const response = await client.get<BrowserSession | null>(
`/browser_sessions/${browserSessionId}`,
);
- const browserSession = response.data;
+ const sessionState = getBrowserSessionState(response.data);
- if (!browserSession || browserSession.completed_at) {
- setHasBrowserSession(false);
- setIsBrowserSessionStarted(false);
- setIsBrowserSessionEnded(true);
- return false;
- }
-
- setHasBrowserSession(true);
- const sessionStarted = Boolean(
- browserSession.started_at || browserSession.browser_address,
- );
- setIsBrowserSessionStarted(sessionStarted);
- return sessionStarted;
+ setHasBrowserSession(sessionState.hasBrowserSession);
+ setIsBrowserSessionStarted(sessionState.isBrowserSessionStarted);
+ setIsBrowserSessionEnded(sessionState.isBrowserSessionEnded);
+ return sessionState.isBrowserSessionStarted;
} catch (error) {
setHasBrowserSession(false);
setIsBrowserSessionStarted(false);
@@ -200,7 +217,8 @@ function BrowserStream({
return false;
}
},
- enabled: entity === "browserSession" && !!browserSessionId,
+ enabled:
+ entity === "browserSession" && !!browserSessionId && !parentSessionState,
refetchInterval: (query) =>
query.state.status === "error" && isForbiddenError(query.state.error)
? false
@@ -209,9 +227,18 @@ function BrowserStream({
: 1000,
});
- const [hasBrowserSession, setHasBrowserSession] = useState(true); // be optimistic
- const [isBrowserSessionStarted, setIsBrowserSessionStarted] = useState(false);
- const [isBrowserSessionEnded, setIsBrowserSessionEnded] = useState(false);
+ const [polledHasBrowserSession, setHasBrowserSession] = useState(true); // be optimistic
+ const [polledIsBrowserSessionStarted, setIsBrowserSessionStarted] =
+ useState(false);
+ const [polledIsBrowserSessionEnded, setIsBrowserSessionEnded] =
+ useState(false);
+ const hasBrowserSession =
+ parentSessionState?.hasBrowserSession ?? polledHasBrowserSession;
+ const isBrowserSessionStarted =
+ parentSessionState?.isBrowserSessionStarted ??
+ polledIsBrowserSessionStarted;
+ const isBrowserSessionEnded =
+ parentSessionState?.isBrowserSessionEnded ?? polledIsBrowserSessionEnded;
const [hasGivenUp, setHasGivenUp] = useState(false);
const [userIsControlling, setUserIsControlling] = useState(false);
const [vncDisconnectedTrigger, setVncDisconnectedTrigger] = useState(0);
diff --git a/skyvern-frontend/src/components/EngineSelector.test.tsx b/skyvern-frontend/src/components/EngineSelector.test.tsx
index 4db879b96..d3f3cdd82 100644
--- a/skyvern-frontend/src/components/EngineSelector.test.tsx
+++ b/skyvern-frontend/src/components/EngineSelector.test.tsx
@@ -102,13 +102,10 @@ describe("RunEngineSelector", () => {
expect(onChange.mock.calls).toEqual([[RunEngine.SkyvernV3], [null]]);
});
- test.each([
- [null, /follows engine routing/],
- [RunEngine.SkyvernV3, /runs on Skyvern 3\.0/],
- ])(
- "Default on a workflow whose effective default is %s says %s",
- (effectiveDefaultEngine, hint) => {
- render(
+ test.each([null, RunEngine.SkyvernV3])(
+ "labels Default with no hint when the effective default is %s",
+ (effectiveDefaultEngine) => {
+ const { container } = render(
<RunEngineSelector
value={null}
onChange={() => {}}
@@ -116,8 +113,14 @@ describe("RunEngineSelector", () => {
/>,
);
- expect(screen.getAllByText("Default").length).toBeGreaterThan(0);
- expect(screen.getAllByText(hint).length).toBeGreaterThan(0);
+ const defaultItem = screen
+ .getAllByRole("button")
+ .filter((button) => button.textContent?.includes("Default"));
+ expect(defaultItem.map((button) => button.textContent)).toEqual([
+ "Default",
+ "Default",
+ ]);
+ expect(container.textContent).not.toMatch(/routing|runs on/i);
},
);
diff --git a/skyvern-frontend/src/components/EngineSelector.tsx b/skyvern-frontend/src/components/EngineSelector.tsx
index 899e9ed32..024f1d82e 100644
--- a/skyvern-frontend/src/components/EngineSelector.tsx
+++ b/skyvern-frontend/src/components/EngineSelector.tsx
@@ -73,21 +73,10 @@ const defaultEngines: Array<RunEngine> = [
RunEngine.AnthropicCua,
];
-function defaultOption(effectiveDefaultEngine: RunEngine | null): EngineOption {
- return {
- value: DEFAULT_ENGINE_VALUE,
- label: (
- <span className="flex items-center gap-1.5">
- <span>Default</span>
- <span className="text-xs text-muted-foreground">
- {effectiveDefaultEngine === RunEngine.SkyvernV3
- ? "runs on Skyvern 3.0"
- : "follows engine routing"}
- </span>
- </span>
- ),
- };
-}
+const defaultOption: EngineOption = {
+ value: DEFAULT_ENGINE_VALUE,
+ label: "Default",
+};
function RunEngineSelector({
value,
@@ -103,7 +92,7 @@ function RunEngineSelector({
const visibleEngines =
value && !engines.includes(value) ? [...engines, value] : engines;
const options: Array<EngineOption> = [
- defaultOption(effectiveDefaultEngine),
+ defaultOption,
...engineOptions.filter((opt) => visibleEngines.includes(opt.value)),
];
const selectValue = value ?? DEFAULT_ENGINE_VALUE;
diff --git a/skyvern-frontend/src/routes/browserSessions/BrowserSession.tsx b/skyvern-frontend/src/routes/browserSessions/BrowserSession.tsx
index ac441dbe4..42e7146ad 100644
--- a/skyvern-frontend/src/routes/browserSessions/BrowserSession.tsx
+++ b/skyvern-frontend/src/routes/browserSessions/BrowserSession.tsx
@@ -374,6 +374,7 @@ function BrowserSession() {
!vncFailed && (
<BrowserStream
browserSessionId={browserSessionId}
+ browserSession={browserSession}
interactive={false}
showControlButtons={true}
isVisible={activeTab === "stream"}
diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx
index 36926cbe6..31ece3b8c 100644
--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.test.tsx
@@ -737,6 +737,39 @@ describe("PromptBox", () => {
expect(body.request.url).toBe("https://google.com");
});
+ test.each([
+ { minimal: false, coarsePointer: false, submits: true },
+ { minimal: false, coarsePointer: true, submits: false },
+ { minimal: true, coarsePointer: false, submits: true },
+ { minimal: true, coarsePointer: true, submits: false },
+ ])(
+ "Enter submits unless on a touch keyboard; Shift+Enter never does (minimal=$minimal, coarse=$coarsePointer)",
+ async ({ minimal, coarsePointer, submits }) => {
+ vi.stubGlobal("matchMedia", (query: string) => ({
+ matches: query === "(any-pointer: fine)" ? !coarsePointer : true,
+ }));
+ mockPost.mockResolvedValue({
+ data: {
+ workflow_permanent_id: "wpid_1",
+ workflow_definition: { blocks: [] },
+ },
+ });
+ renderPromptBox(false, undefined, minimal);
+ const textarea = document.getElementById("discover-prompt-input")!;
+ fireEvent.change(textarea, { target: { value: "Visit the docs" } });
+
+ fireEvent.keyDown(textarea, { key: "Enter", shiftKey: true });
+ fireEvent.keyDown(textarea, { key: "Enter", isComposing: true });
+ fireEvent.keyDown(textarea, { key: "Enter" });
+
+ if (submits) {
+ await waitFor(() => expect(mockPost).toHaveBeenCalledTimes(1));
+ } else {
+ expect(mockPost).not.toHaveBeenCalled();
+ }
+ },
+ );
+
test("sends settings changed in the gear popover and lists them under the prompt", async () => {
mockPost.mockResolvedValue({
data: {
diff --git a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
index c98dce236..c4e2ce069 100644
--- a/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
+++ b/skyvern-frontend/src/routes/tasks/create/PromptBox.tsx
@@ -39,6 +39,7 @@ import { AxiosError, type AxiosResponse } from "axios";
import {
forwardRef,
type ForwardedRef,
+ type KeyboardEvent,
type ReactNode,
useCallback,
useEffect,
@@ -625,6 +626,112 @@ function PromptBoxImpl(
generateWorkflowMutation.mutate({ prompt, attempt });
};
+ const handlePromptKeyDown = (e: KeyboardEvent<HTMLTextAreaElement>) => {
+ // With no precise pointer (a phone) assume an on-screen keyboard, which has no Shift+Enter, so Return stays a
+ // newline and the send button submits. A tablet with a keyboard but no trackpad falls in this bucket too.
+ if (
+ e.key !== "Enter" ||
+ e.shiftKey ||
+ e.nativeEvent.isComposing ||
+ !window.matchMedia?.("(any-pointer: fine)").matches
+ ) {
+ return;
+ }
+ e.preventDefault();
+ if (prompt.trim()) {
+ submitPrompt({ prompt, attribution: exampleAttribution });
+ }
+ };
+
+ const attachmentChips =
+ attachedFiles.length > 0 ? (
+ <div className="flex flex-wrap gap-1.5 px-1 pt-2">
+ {attachedFiles.map((file) => (
+ <span
+ key={file.file_id}
+ className="inline-flex items-center gap-1.5 rounded-md border border-input bg-slate-elevation2 px-2 py-1 text-xs text-foreground"
+ >
+ <FileTextIcon aria-hidden="true" className="size-3.5" />
+ <span className="max-w-[16rem] truncate">{file.filename}</span>
+ <button
+ type="button"
+ aria-label={`Remove ${file.filename}`}
+ className="text-muted-foreground hover:text-foreground"
+ onClick={() =>
+ setAttachedFiles((current) =>
+ current.filter((f) => f.file_id !== file.file_id),
+ )
+ }
+ >
+ <Cross2Icon aria-hidden="true" className="size-3" />
+ </button>
+ </span>
+ ))}
+ </div>
+ ) : null;
+
+ const fileInput = (
+ <input
+ ref={fileInputRef}
+ type="file"
+ className="hidden"
+ aria-label="Upload document"
+ onChange={(event) => {
+ const file = event.target.files?.[0];
+ if (file) {
+ uploadDocumentMutation.mutate(file);
+ }
+ event.target.value = "";
+ }}
+ />
+ );
+
+ const addMenu = (
+ <DropdownMenu
+ onOpenChange={(open) => {
+ if (open) HomeTelemetry.addMenuOpened();
+ }}
+ >
+ <DropdownMenuTrigger asChild>
+ <button
+ type="button"
+ aria-label="Add to prompt"
+ disabled={isSubmitting}
+ className="flex size-8 items-center justify-center rounded-lg border border-input text-foreground transition-colors hover:bg-muted disabled:opacity-50"
+ >
+ {uploadDocumentMutation.isPending || recordTaskMutation.isPending ? (
+ <ReloadIcon className="size-4 animate-spin" />
+ ) : (
+ <PlusIcon aria-hidden="true" className="size-4" />
+ )}
+ </button>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent align="start">
+ {enableCopilotHandoff ? (
+ <DropdownMenuItem
+ disabled={attachedFiles.length >= MAX_HOME_ATTACHMENTS}
+ onSelect={() => {
+ HomeTelemetry.uploadDocumentSelected();
+ fileInputRef.current?.click();
+ }}
+ >
+ <UploadIcon className="mr-2 size-4" />
+ Upload document
+ </DropdownMenuItem>
+ ) : null}
+ <DropdownMenuItem
+ onSelect={() => {
+ HomeTelemetry.recordTaskSelected();
+ recordTaskMutation.mutate();
+ }}
+ >
+ <VideoIcon className="mr-2 size-4" />
+ Record task
+ </DropdownMenuItem>
+ </DropdownMenuContent>
+ </DropdownMenu>
+ );
+
if (!minimal) {
return (
<div className="relative isolate flex flex-col items-center pb-4 pt-12 md:pt-24">
@@ -653,9 +760,19 @@ function PromptBoxImpl(
className="max-h-[14rem] min-h-[6rem] resize-none overflow-y-auto border-0 bg-transparent px-5 pb-1.5 pt-[18px] text-base leading-6 text-foreground shadow-none placeholder:text-muted-foreground hover:border-0 focus-visible:ring-0 md:text-[15px]"
value={prompt}
onChange={(e) => updatePrompt(e.target.value)}
+ onKeyDown={handlePromptKeyDown}
placeholder="Enter your prompt..."
/>
+ {enableCopilotHandoff ? (
+ <div className="px-5">{attachmentChips}</div>
+ ) : null}
<div className="flex items-center gap-1 px-2.5 pb-2.5 pt-2">
+ {enableCopilotHandoff ? (
+ <>
+ {addMenu}
+ {fileInput}
+ </>
+ ) : null}
<SpeechInputButton
isSupported={isSpeechSupported}
isListening={isSpeechListening}
@@ -796,94 +913,14 @@ function PromptBoxImpl(
className="max-h-[8rem] min-h-[4rem] resize-none overflow-y-auto border-0 bg-transparent px-3 py-3 leading-5 text-foreground shadow-none placeholder:text-muted-foreground hover:border-0 focus-visible:ring-0"
value={prompt}
onChange={(e) => updatePrompt(e.target.value)}
+ onKeyDown={handlePromptKeyDown}
onFocus={() => setPromptTouched(true)}
cycling={!promptTouched}
/>
- {attachedFiles.length > 0 ? (
- <div className="flex flex-wrap gap-1.5 px-1 pt-2">
- {attachedFiles.map((file) => (
- <span
- key={file.file_id}
- className="inline-flex items-center gap-1.5 rounded-md border border-input bg-slate-elevation2 px-2 py-1 text-xs text-foreground"
- >
- <FileTextIcon aria-hidden="true" className="size-3.5" />
- <span className="max-w-[16rem] truncate">
- {file.filename}
- </span>
- <button
- type="button"
- aria-label={`Remove ${file.filename}`}
- className="text-muted-foreground hover:text-foreground"
- onClick={() =>
- setAttachedFiles((current) =>
- current.filter((f) => f.file_id !== file.file_id),
- )
- }
- >
- <Cross2Icon aria-hidden="true" className="size-3" />
- </button>
- </span>
- ))}
- </div>
- ) : null}
+ {attachmentChips}
<div className="flex items-center gap-1 pt-2">
- <DropdownMenu
- onOpenChange={(open) => {
- if (open) HomeTelemetry.addMenuOpened();
- }}
- >
- <DropdownMenuTrigger asChild>
- <button
- type="button"
- aria-label="Add to prompt"
- disabled={isSubmitting}
- className="flex size-8 items-center justify-center rounded-lg border border-input text-foreground transition-colors hover:bg-muted disabled:opacity-50"
- >
- {uploadDocumentMutation.isPending ||
- recordTaskMutation.isPending ? (
- <ReloadIcon className="size-4 animate-spin" />
- ) : (
- <PlusIcon aria-hidden="true" className="size-4" />
- )}
- </button>
- </DropdownMenuTrigger>
- <DropdownMenuContent align="start">
- {enableCopilotHandoff ? (
- <DropdownMenuItem
- disabled={attachedFiles.length >= MAX_HOME_ATTACHMENTS}
- onSelect={() => {
- HomeTelemetry.uploadDocumentSelected();
- fileInputRef.current?.click();
- }}
- >
- <UploadIcon className="mr-2 size-4" />
- Upload document
- </DropdownMenuItem>
- ) : null}
- <DropdownMenuItem
- onSelect={() => {
- HomeTelemetry.recordTaskSelected();
- recordTaskMutation.mutate();
- }}
- >
- <VideoIcon className="mr-2 size-4" />
- Record task
- </DropdownMenuItem>
- </DropdownMenuContent>
- </DropdownMenu>
- <input
- ref={fileInputRef}
- type="file"
- className="hidden"
- aria-label="Upload document"
- onChange={(event) => {
- const file = event.target.files?.[0];
- if (file) {
- uploadDocumentMutation.mutate(file);
- }
- event.target.value = "";
- }}
- />
+ {addMenu}
+ {fileInput}
<div className="ml-auto flex items-center gap-1">
<SpeechInputButton
isSupported={isSpeechSupported}
diff --git a/skyvern-frontend/src/routes/workflows/ImportWorkflowButton.tsx b/skyvern-frontend/src/routes/workflows/ImportWorkflowButton.tsx
index 4875c63fc..bfea108dd 100644
--- a/skyvern-frontend/src/routes/workflows/ImportWorkflowButton.tsx
+++ b/skyvern-frontend/src/routes/workflows/ImportWorkflowButton.tsx
@@ -86,7 +86,7 @@ function ImportWorkflowButton({
}
await client.post<string, { data: WorkflowApiResponse }>(
"/workflows",
- stripLegacyEngineFromYaml(yaml),
+ yaml,
{
headers: {
"Content-Type": "text/plain",
@@ -166,10 +166,22 @@ function ImportWorkflowButton({
if (files.length === 0) {
return;
}
+ const stripped = files.map((f) => ({
+ ...f,
+ ...stripLegacyEngineFromYaml(f.yaml),
+ }));
const results = await Promise.all(
- files.map((f) => createWorkflowFromYaml(f.yaml, f.title ?? f.fileName)),
+ stripped.map((f) =>
+ createWorkflowFromYaml(f.yaml, f.title ?? f.fileName),
+ ),
);
const successCount = results.filter(Boolean).length;
+ // One toast shows at a time, so the reset notice rides on the success toast.
+ const resetNotice = stripped.some(
+ (f, i) => f.strippedLegacyEngines && results[i],
+ )
+ ? " Legacy engine settings were reset to Default."
+ : "";
if (successCount > 0) {
queryClient.invalidateQueries({ queryKey: ["workflows"] });
queryClient.invalidateQueries({ queryKey: ["folders"] });
@@ -180,9 +192,10 @@ function ImportWorkflowButton({
? "Agent imported"
: `${successCount} agents imported`,
description:
- successCount === files.length
- ? "Successfully imported all agents"
- : `${successCount} of ${files.length} agents imported successfully`,
+ (successCount === files.length
+ ? "Successfully imported all agents."
+ : `${successCount} of ${files.length} agents imported successfully.`) +
+ resetNotice,
});
}
};
diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
index 8c676055a..24b671329 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts
@@ -802,6 +802,7 @@ const TOOL_CALL_KINDS: Record<string, ToolCallKind> = {
search_web: "other",
list_org_workflows: "other",
get_org_workflow: "other",
+ list_workflow_runs: "other",
list_workflow_schedules: "other",
get_workflow_schedule: "other",
create_workflow_schedule: "other",
diff --git a/skyvern-frontend/src/routes/workflows/editor/helpContent.ts b/skyvern-frontend/src/routes/workflows/editor/helpContent.ts
index a052a8d2c..2c9e2db05 100644
--- a/skyvern-frontend/src/routes/workflows/editor/helpContent.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/helpContent.ts
@@ -30,7 +30,7 @@ export const baseHelpTooltipContent = {
includeActionHistoryInVerification:
"Include the action history in the completion verification.",
engine:
- "Default follows the workflow's engine routing, or runs on Skyvern 3.0 for new workflows. Choosing an engine pins this block to it.",
+ "Default lets Skyvern choose the engine. Choosing an engine pins this block to it.",
} as const;
export const basePlaceholderContent = {
diff --git a/skyvern-frontend/src/routes/workflows/importWorkflowYaml.test.ts b/skyvern-frontend/src/routes/workflows/importWorkflowYaml.test.ts
index 7d0875cce..c8b5edc01 100644
--- a/skyvern-frontend/src/routes/workflows/importWorkflowYaml.test.ts
+++ b/skyvern-frontend/src/routes/workflows/importWorkflowYaml.test.ts
@@ -187,43 +187,6 @@ describe("extractTitleFromYaml", () => {
});
describe("stripLegacyEngineFromYaml", () => {
- const workflow = {
- title: "W",
- workflow_definition: {
- blocks: [
- { block_type: "navigation", label: "a", engine: "skyvern-1.0" },
- { block_type: "navigation", label: "b", engine: "skyvern-2.0" },
- { block_type: "navigation", label: "c", engine: "skyvern-3.0" },
- { block_type: "navigation", label: "d" },
- {
- block_type: "for_loop",
- label: "e",
- loop_blocks: [
- { block_type: "navigation", label: "f", engine: "skyvern-1.0" },
- { block_type: "navigation", label: "g", engine: "skyvern-3.0" },
- ],
- },
- ],
- },
- };
-
- it("unsets skyvern-1.0 at any depth and keeps every other engine", () => {
- const out = parseYAML(stripLegacyEngineFromYaml(convertToYAML(workflow)))
- .workflow_definition.blocks;
- expect(out[0]).not.toHaveProperty("engine");
- expect(out[1].engine).toBe("skyvern-2.0");
- expect(out[2].engine).toBe("skyvern-3.0");
- expect(out[3]).not.toHaveProperty("engine");
- expect(out[4].loop_blocks[0]).not.toHaveProperty("engine");
- expect(out[4].loop_blocks[1].engine).toBe("skyvern-3.0");
- });
-
- it("returns a file with no legacy engine byte-for-byte", () => {
- const clean =
- "# kept\ntitle: t\nworkflow_definition:\n blocks:\n - block_type: navigation\n label: a\n engine: skyvern-3.0\n";
- expect(stripLegacyEngineFromYaml(clean)).toBe(clean);
- });
-
const lines = [
"# exported",
"title: t",
@@ -240,7 +203,7 @@ describe("stripLegacyEngineFromYaml", () => {
" navigation_goal: 1e3",
" complete_criterion: 2001-12-14 21:59:43.10 -05:00",
" - engine: skyvern-1.0",
- " block_type: navigation",
+ " block_type: extraction",
" label: b",
" - block_type: for_loop",
" label: c",
@@ -248,40 +211,120 @@ describe("stripLegacyEngineFromYaml", () => {
" - block_type: navigation",
" label: d",
" engine: 'skyvern-1.0'",
- " - block_type: navigation",
+ " - block_type: wait",
" label: e",
- " engine: skyvern-3.0",
- " - block_type: navigation",
+ " wait_sec: 1",
+ " - block_type: code",
" label: f",
- " engine: null",
+ " code: x",
"",
];
+ const legacy = lines.join("\n");
const withoutLegacy = [...lines];
withoutLegacy.splice(22, 1);
- withoutLegacy.splice(14, 2, " - block_type: navigation");
+ withoutLegacy.splice(14, 2, " - block_type: extraction");
withoutLegacy.splice(11, 1);
- it("removes only the skyvern-1.0 lines, at any depth, and leaves every other byte", () => {
- expect(stripLegacyEngineFromYaml(lines.join("\n"))).toBe(
- withoutLegacy.join("\n"),
- );
+ it("removes only the skyvern-1.0 lines of a legacy export, at any depth, and leaves every other byte", () => {
+ expect(stripLegacyEngineFromYaml(legacy)).toEqual({
+ yaml: withoutLegacy.join("\n"),
+ strippedLegacyEngines: true,
+ });
});
- it("returns a file with no skyvern-1.0 byte-for-byte, values and comments included", () => {
- const clean = withoutLegacy.join("\n");
- expect(clean).toContain("engine: null");
- expect(stripLegacyEngineFromYaml(clean)).toBe(clean);
+ it("strips a legacy export whose validation block predates its engine key", () => {
+ const bare = " - block_type: validation\n label: g\n";
+ expect(stripLegacyEngineFromYaml(legacy + bare)).toEqual({
+ yaml: withoutLegacy.join("\n") + bare,
+ strippedLegacyEngines: true,
+ });
+ });
+
+ it.each(
+ [
+ "task",
+ "navigation",
+ "action",
+ "extraction",
+ "login",
+ "file_download",
+ "validation",
+ ].flatMap((type) => [
+ [type, "null"],
+ [type, "skyvern-3.0"],
+ ]),
+ )(
+ "keeps skyvern-1.0 as a pin beside a %s block with engine %s",
+ (type, engine) => {
+ const yaml = `${legacy} - block_type: ${type}\n engine: ${engine}\n`;
+ expect(stripLegacyEngineFromYaml(yaml)).toEqual({
+ yaml,
+ strippedLegacyEngines: false,
+ });
+ },
+ );
+
+ it.each([
+ [
+ "a task block without an engine",
+ legacy.replace(" engine: skyvern-1.0 # legacy\n", ""),
+ ],
+ [
+ "a task block pinned to another engine",
+ legacy.replace("'skyvern-1.0'", "skyvern-3.0"),
+ ],
+ [
+ "a task block with a null engine",
+ legacy.replace("'skyvern-1.0'", "null"),
+ ],
+ ["no engine at all", withoutLegacy.join("\n")],
+ [
+ "only blocks that carry no engine",
+ "workflow_definition:\n blocks:\n - block_type: wait\n - block_type: code\n",
+ ],
+ ])("keeps skyvern-1.0 as a pin in a file with %s", (_, yaml) => {
+ expect(stripLegacyEngineFromYaml(yaml)).toEqual({
+ yaml,
+ strippedLegacyEngines: false,
+ });
});
it("leaves a flow-style block holding skyvern-1.0 unchanged rather than rewrite it", () => {
const flow =
"workflow_definition:\n blocks:\n - {label: a, engine: skyvern-1.0, goal: Y}\n";
- expect(stripLegacyEngineFromYaml(flow)).toBe(flow);
+ expect(stripLegacyEngineFromYaml(flow).yaml).toBe(flow);
+ });
+
+ it("leaves a file with merged or aliased blocks unchanged, since their inherited fields are unread", () => {
+ const merged = [
+ "title: t",
+ "defaults: &nav",
+ " block_type: navigation",
+ " engine: skyvern-3.0",
+ "workflow_definition:",
+ " blocks:",
+ " - <<: *nav",
+ " label: a",
+ " - block_type: navigation",
+ " label: b",
+ " engine: skyvern-1.0",
+ "",
+ ].join("\n");
+ const aliased = merged.replace(
+ " - <<: *nav\n label: a",
+ " - *nav",
+ );
+ for (const file of [merged, aliased]) {
+ expect(stripLegacyEngineFromYaml(file)).toEqual({
+ yaml: file,
+ strippedLegacyEngines: false,
+ });
+ }
});
it("returns unparseable text unchanged", () => {
const bad = "title: [unclosed\n engine: skyvern-1.0";
- expect(stripLegacyEngineFromYaml(bad)).toBe(bad);
+ expect(stripLegacyEngineFromYaml(bad).yaml).toBe(bad);
});
});
diff --git a/skyvern-frontend/src/routes/workflows/importWorkflowYaml.ts b/skyvern-frontend/src/routes/workflows/importWorkflowYaml.ts
index a61965fd8..e0c7dbfc7 100644
--- a/skyvern-frontend/src/routes/workflows/importWorkflowYaml.ts
+++ b/skyvern-frontend/src/routes/workflows/importWorkflowYaml.ts
@@ -124,17 +124,34 @@ export function extractTitleFromYaml(yaml: string): string | null {
type Span = [number, number];
+// The block types whose YAML carries an engine. The old editor wrote skyvern-1.0 on every one of them.
+const ENGINE_BLOCK_TYPES = new Set([
+ "task",
+ "navigation",
+ "action",
+ "extraction",
+ "login",
+ "file_download",
+ "validation",
+]);
+
// Returns the byte span of each `engine: skyvern-1.0` line, or null when one sits where it cannot be cut
-// without rewriting its neighbours (flow style, or sharing a line with other content).
+// without rewriting its neighbours (flow style, or sharing a line with other content), or when an
+// engine-bearing block lacks skyvern-1.0, which marks a file from the new editor where 1.0 is a pin.
function legacyEngineSpans(text: string, blocks: unknown): Span[] | null {
if (!isSeq(blocks)) {
return [];
}
const spans: Span[] = [];
for (const block of blocks.items) {
- if (!isMap(block)) {
- continue;
+ // An aliased block or a merge key hides fields this pass would read, so such a file is left as written.
+ if (
+ !isMap(block) ||
+ block.items.some((pair) => isScalar(pair.key) && pair.key.value === "<<")
+ ) {
+ return null;
}
+ const spansBefore = spans.length;
for (const [index, pair] of block.items.entries()) {
const { key, value } = pair;
if (
@@ -174,6 +191,16 @@ function legacyEngineSpans(text: string, blocks: unknown): Span[] | null {
}
spans.push([key.range[0], next.range[0]]);
}
+ const blockType = block.get("block_type");
+ if (
+ typeof blockType === "string" &&
+ ENGINE_BLOCK_TYPES.has(blockType) &&
+ // Validation blocks were exported without an engine key until #16538, so there an absent key is no evidence.
+ (block.has("engine") || blockType !== "validation") &&
+ spans.length === spansBefore
+ ) {
+ return null;
+ }
const nested = legacyEngineSpans(text, block.get("loop_blocks", true));
if (nested === null) {
return null;
@@ -183,24 +210,28 @@ function legacyEngineSpans(text: string, blocks: unknown): Span[] | null {
return spans;
}
+export type StrippedYaml = { yaml: string; strippedLegacyEngines: boolean };
+
// Cuts only the legacy engine lines out of the original text, so every other byte reaches the backend as written.
-export function stripLegacyEngineFromYaml(yaml: string): string {
+export function stripLegacyEngineFromYaml(yaml: string): StrippedYaml {
+ const unchanged = { yaml, strippedLegacyEngines: false };
try {
const document = parseDocument(yaml);
if (document.errors.length > 0 || !isMap(document.contents)) {
- return yaml;
+ return unchanged;
}
const definition = document.contents.get("workflow_definition", true);
const spans = isMap(definition)
? legacyEngineSpans(yaml, definition.get("blocks", true))
: null;
- if (!spans) {
- return yaml;
+ if (!spans || spans.length === 0) {
+ return unchanged;
}
- return spans
+ const stripped = spans
.sort((a, b) => b[0] - a[0])
.reduce((text, [from, to]) => text.slice(0, from) + text.slice(to), yaml);
+ return { yaml: stripped, strippedLegacyEngines: true };
} catch {
- return yaml;
+ return unchanged;
}
}
diff --git a/skyvern/cli/mcp_tools/workflow.py b/skyvern/cli/mcp_tools/workflow.py
index 248ee3ba4..206ef2a53 100644
--- a/skyvern/cli/mcp_tools/workflow.py
+++ b/skyvern/cli/mcp_tools/workflow.py
@@ -350,6 +350,9 @@ def _serialize_run_summary(run: Any) -> dict[str, Any]:
"run_id": run_id,
"status": str(_get_value(run, "status")) if _get_value(run, "status") is not None else None,
"run_type": str(run_type) if run_type is not None else None,
+ "created_at": _jsonable(_get_value(run, "created_at")),
+ "trigger_type": _jsonable(_get_value(run, "trigger_type")),
+ "copilot_session_id": _get_value(run, "copilot_session_id"),
"artifact_summary": _summarize_artifacts(run, output_stats),
"output_summary": output_summary,
}
@@ -1648,16 +1651,20 @@ async def skyvern_workflow_run_list(
with Timer() as timer:
try:
- requested_page_size = page_size
- runs = await list_workflow_runs_raw(
+ list_runs = functools.partial(
+ list_workflow_runs_raw,
workflow_id,
- page=page,
- page_size=requested_page_size + 1,
status=status,
search_key=search_key,
error_code=error_code,
include_child_runs=include_child_runs,
)
+ runs = await list_runs(page=page, page_size=page_size)
+ has_more = False
+ if len(runs) == page_size:
+ # The route offsets by (page - 1) * page_size, so this one-row page is the next page's first run.
+ next_run = await list_runs(page=page * page_size + 1, page_size=1)
+ has_more = bool(next_run)
timer.mark("sdk")
except NotFoundError:
return make_result(
@@ -1678,9 +1685,6 @@ async def skyvern_workflow_run_list(
error=make_error(ErrorCode.API_ERROR, str(e), "Check your API key and workflow run filters"),
)
- has_more = len(runs) > page_size
- runs = runs[:page_size]
-
return make_result(
"skyvern_workflow_run_list",
data={
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 8a2b85c1e..d17d3334f 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -192,6 +192,7 @@ from skyvern.forge.taskv3.goal_check import (
GoalJudge,
goal_check_eligible,
)
+from skyvern.forge.taskv3.goal_composition import CodeProgressRecord
from skyvern.forge.taskv3.loop import LoopOutcome, RoundAction
from skyvern.forge.taskv3.pre_submit_capture import PreSubmitCaptureRing, is_run_sampled, pre_submit_screenshot
from skyvern.forge.taskv3.run_arms import (
@@ -2216,6 +2217,7 @@ class ForgeAgent:
workflow_owned_recovery: bool = False,
recovery_credential_parameter_keys: list[str] | None = None,
recovery_release_parameter_keys: list[str] | None = None,
+ recovery_code_progress: CodeProgressRecord | None = None,
) -> tuple[Step, Task]:
"""Run a whole task via the native Task V3 tool-loop (one persistent conversation).
@@ -2437,6 +2439,7 @@ class ForgeAgent:
framing=framing,
block_context_section=block_context_section,
page_data_note=page_data_note,
+ code_progress=recovery_code_progress,
),
)
@@ -3652,6 +3655,7 @@ class ForgeAgent:
workflow_owned_recovery: bool = False,
recovery_credential_parameter_keys: list[str] | None = None,
recovery_release_parameter_keys: list[str] | None = None,
+ recovery_code_progress: CodeProgressRecord | None = None,
cua_response: OpenAIResponse | None = None,
llm_caller: LLMCaller | None = None,
download_baseline_files: list[str] | None = None,
@@ -3873,6 +3877,7 @@ class ForgeAgent:
workflow_owned_recovery=workflow_owned_recovery,
recovery_credential_parameter_keys=recovery_credential_parameter_keys,
recovery_release_parameter_keys=recovery_release_parameter_keys,
+ recovery_code_progress=recovery_code_progress,
)
finally:
await app.ARTIFACT_MANAGER.flush_step_archive(step.step_id)
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index c298e87b9..986fb1c6e 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -74,6 +74,7 @@ from skyvern.forge.sdk.services import (
sftp_service,
)
from skyvern.forge.sdk.services.credentials import AuthenticatorTotpParseResult
+from skyvern.forge.sdk.services.request_principal import BearerIdentity
from skyvern.forge.sdk.trace import traced
from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, BlockTypeVar
from skyvern.forge.sdk.workflow.retry_policy import WORKFLOW_WEBHOOK_HTTP_TIMEOUT_SECONDS
@@ -2964,6 +2965,10 @@ class AgentFunction:
"""Return whether the user belongs to the organization, or None when membership cannot be determined."""
return None
+ async def resolve_bearer_identity(self, bearer_token: str, organization_id: str) -> BearerIdentity | None:
+ """Return the user the bearer proves for this organization, or None without an identity provider or proof."""
+ return None
+
async def on_workflow_saved(
self,
organization_id: str,
diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.py
index 983a39754..38fd05bad 100644
--- a/skyvern/forge/request_logging.py
+++ b/skyvern/forge/request_logging.py
@@ -30,6 +30,8 @@ if typing.TYPE_CHECKING: # pragma: no cover - import only for type hints
from starlette.requests import Request
from starlette.types import Message, Receive, Scope, Send
+ from skyvern.forge.sdk.services.request_principal import RequestPrincipal
+
LOG = structlog.get_logger()
_SENSITIVE_ENDPOINTS = {
@@ -89,6 +91,7 @@ class _RequestIdentity:
organization_id: str | None = None
organization_name: str | None = None
org_age: int | None = None
+ principal: RequestPrincipal | None = None
_request_identity: ContextVar[_RequestIdentity | None] = ContextVar("raw_request_identity", default=None)
@@ -116,17 +119,29 @@ def set_request_organization(
identity.org_age = org_age
-def _organization_log_fields() -> dict[str, str | int]:
+def set_request_principal(principal: RequestPrincipal) -> None:
+ identity = _request_identity.get()
+ if identity is not None:
+ identity.principal = principal
+
+
+def _identity_log_fields() -> dict[str, str | int | None]:
identity = _request_identity.get()
if identity is None:
return {}
- fields: dict[str, str | int] = {}
+ fields: dict[str, str | int | None] = {}
if identity.organization_id:
fields["organization_id"] = identity.organization_id
if identity.organization_name:
fields["organization_name"] = identity.organization_name
if identity.org_age is not None:
fields["org_age"] = identity.org_age
+ if identity.principal is not None:
+ # Explicit nulls: "resolved, no user" must stay distinguishable from an unauthenticated request.
+ fields["auth_kind"] = identity.principal.auth_kind.value
+ fields["user_id"] = identity.principal.user_id
+ fields["org_role"] = identity.principal.org_role
+ fields["org_role_claim"] = identity.principal.org_role_claim
return fields
@@ -250,7 +265,7 @@ def _log_unhandled_request(
headers=headers,
duration_seconds=time.monotonic() - start_time,
**(exception_log_fields(exc) if exc is not None else {}),
- **_organization_log_fields(),
+ **_identity_log_fields(),
)
except Exception:
pass
@@ -287,7 +302,7 @@ def _log_request(
# backwards-compat: keep error_body for existing Datadog queries
error_body=response_body if status_code >= 400 else None,
duration_seconds=time.monotonic() - start_time,
- **_organization_log_fields(),
+ **_identity_log_fields(),
)
except Exception:
pass
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 1117d0a12..53f741f56 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -1735,10 +1735,15 @@ def _rewrite_failed_test_response(user_response: str, ctx: CopilotContext) -> st
)
# No run row means nothing executed, so claiming the draft was tested is false.
- if ctx.last_failure_category_top == "UNRECOVERABLE_TOOL_ERROR" and not ctx.dispatched_run_ids_this_turn:
+ if (
+ ctx.last_failure_category_top == "UNRECOVERABLE_TOOL_ERROR" and not ctx.dispatched_run_ids_this_turn
+ ) or ctx.last_test_run_started is False:
+ not_started_reason = _normalize_failure_reason(ctx.last_test_failure_reason)
+ if not not_started_reason.endswith("..."):
+ not_started_reason = not_started_reason.rstrip(".")
return (
f"I created {draft_phrase}, but I couldn't start a test run: "
- f"{_normalize_failure_reason(ctx.last_test_failure_reason)}. "
+ f"{not_started_reason}. "
f"Nothing was executed, so the draft is unverified.{keep_draft_affordance}"
)
@@ -3173,7 +3178,7 @@ def _recorded_failure_reply(
# overall_status="failed" even though execution never started.
run_created = bool(getattr(diagnosis_input, "workflow_run_id", None))
if getattr(ctx, "last_workflow", None) is not None:
- if unrecoverable_stop and not run_created:
+ if (unrecoverable_stop and not run_created) or ctx.last_test_run_started is False:
return f"I built {block_phrase}, but I couldn't start a test run: {reason}.{status_sentence}"
if test_attempted and test_failed and not unrecoverable_stop:
return f"I built {block_phrase} and tested it, but the test failed: {reason}.{status_sentence}"
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 94be2b612..8fde3589b 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -1300,6 +1300,8 @@ class CopilotContext(AgentContext):
# Runs this turn actually dispatched. ``last_run_blocks_workflow_run_id`` cannot answer that:
# a repair turn is seeded with the run it was opened about, which this turn never dispatched.
dispatched_run_ids_this_turn: set[str] = field(default_factory=set)
+ # Whether the latest recorded run-tool result created a run row; None until one is recorded.
+ last_test_run_started: bool | None = None
# The files the last recorded worker run published, keyed by that run's id.
delivered_output_files: tuple[str, list[DeliveredOutputFile]] | None = None
# Producers a dispatched run was seeded with and never ran; their values belong to another run.
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 78b0cd95c..101451cb8 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -508,13 +508,17 @@ class SchemaOverlay:
# stripped before the call, so the underlying tool never sees an argument it cannot accept.
copilot_params: dict[str, Any] = field(default_factory=dict)
requires_browser: bool = False
- # Dispatch overwrites workflow_permanent_id with the chat's own, so the model cannot aim the
- # call at any other workflow. Pair it with hiding that param from the schema.
+ # Dispatch overwrites binds_chat_workflow_param with the chat's own workflow, so the model cannot
+ # aim the call at any other workflow. Pair it with hiding that param from the schema.
binds_chat_workflow: bool = False
+ binds_chat_workflow_param: str = "workflow_permanent_id"
# Creates or changes future runs, so a turn without browser authority, which may not start a
# run either, does not see it.
requires_run_authority: bool = False
redacts_sensitive_origin_structured_result: bool = False
+ # Set when the post-hook is what keeps stored values from the model: a crash then withholds the
+ # result instead of falling back to the unfiltered one.
+ post_hook_fails_closed: bool = False
timeout: int | None = None
pre_hook: PreHook | None = None
post_hook: PostHook | None = None
@@ -1759,7 +1763,7 @@ class SkyvernOverlayMCPServer(MCPServer):
mcp_args = _transform_args(arguments, overlay)
if overlay.binds_chat_workflow:
- mcp_args["workflow_permanent_id"] = copilot_ctx.workflow_permanent_id
+ mcp_args[overlay.binds_chat_workflow_param] = copilot_ctx.workflow_permanent_id
if overlay.requires_browser:
phases.enter("session_prepare")
@@ -1950,10 +1954,24 @@ class SkyvernOverlayMCPServer(MCPServer):
_restore_post_hook_context(copilot_ctx, ctx_snapshot)
raise
except Exception:
- # A post-hook enriches evidence only; a crash must not fail the browser action or keep partial credit.
+ # An enrichment hook's crash must not fail the action or keep partial credit.
_restore_post_hook_context(copilot_ctx, ctx_snapshot)
- LOG.warning("MCP post-hook failed; returning base tool result", tool=tool_name)
- copilot_result = base_copilot_result
+ LOG.warning(
+ "MCP post-hook failed",
+ tool=tool_name,
+ fails_closed=overlay.post_hook_fails_closed,
+ )
+ if overlay.post_hook_fails_closed:
+ server_outcome = ", which reported an error" if failed else ", which reported success"
+ copilot_result = {
+ "ok": False,
+ "error": (
+ f"{tool_name} reached the server{server_outcome}, but its result could not "
+ "be prepared for display and is withheld."
+ ),
+ }
+ else:
+ copilot_result = base_copilot_result
# The last disclosure boundary, so it also runs after a crashed post-hook. It fails
# closed when an unexpected producer marks the exact session while an enrichment awaits.
sensitive_result_is_scrubbable = (
diff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.py
index d4b82f15a..fbcf980a1 100644
--- a/skyvern/forge/sdk/copilot/model_telemetry.py
+++ b/skyvern/forge/sdk/copilot/model_telemetry.py
@@ -630,7 +630,7 @@ class CopilotLitellmModel(LitellmModel):
)
else:
result = await super()._fetch_response(
- system_instructions,
+ str(system_instructions) if system_instructions is not None else None,
input,
model_settings,
tools,
diff --git a/skyvern/forge/sdk/copilot/narration.py b/skyvern/forge/sdk/copilot/narration.py
index f67ae0196..d8c442b50 100644
--- a/skyvern/forge/sdk/copilot/narration.py
+++ b/skyvern/forge/sdk/copilot/narration.py
@@ -127,6 +127,7 @@ _TOOL_ACTIVITY_DISPLAY_LABELS = {
"skyvern_tab_switch": "Switching tabs",
"skyvern_tab_close": "Closing a tab",
"list_workflow_schedules": "Checking this workflow's schedules",
+ "list_workflow_runs": "Listing this workflow's runs",
"get_workflow_schedule": "Reading a schedule",
"create_workflow_schedule": "Creating a schedule",
"update_workflow_schedule": "Updating a schedule",
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index e544af781..59d699dea 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -2079,6 +2079,7 @@ def get_skyvern_mcp_alias_map() -> dict[str, str]:
"disable_workflow_schedule": "skyvern_schedule_disable",
"cancel_workflow_schedule": "skyvern_schedule_cancel",
"delete_workflow_schedule": "skyvern_schedule_delete",
+ "list_workflow_runs": "skyvern_workflow_run_list",
}
@@ -2094,6 +2095,10 @@ _SCHEDULE_HINT_REWRITES = (
"Use skyvern_workflow_list to find valid workflow IDs.",
"The workflow open in this chat was not found in this organization.",
),
+ (
+ "Verify the workflow ID with skyvern_workflow_list",
+ "The workflow open in this chat was not found in this organization.",
+ ),
(", or set exact=true to do a full replace", ""),
)
@@ -2210,6 +2215,25 @@ def _mask_schedule_parameter_values(value: object) -> None:
_mask_schedule_parameter_values(item)
+_RUN_LIST_FIELDS = ("run_id", "status", "created_at", "trigger_type", "parent_run_id")
+
+
+async def _workflow_run_list_post_hook(
+ result: dict[str, Any],
+ raw: dict[str, Any],
+ ctx: AgentContext,
+) -> dict[str, Any]:
+ result = await _workflow_schedule_post_hook(result, raw, ctx)
+ data = result.get("data")
+ if isinstance(data, dict) and isinstance(data.get("runs"), list):
+ # A past run's outputs and failure reason can hold secrets this chat never saw, and the scrubber
+ # only knows values it has seen, so a bulk listing leaves them out.
+ data["runs"] = [
+ {key: run[key] for key in _RUN_LIST_FIELDS if key in run} for run in data["runs"] if isinstance(run, dict)
+ ]
+ return result
+
+
def _workflow_schedule_overlay(
description: str,
*,
@@ -2224,6 +2248,7 @@ def _workflow_schedule_overlay(
requires_run_authority=mutates,
pre_hook=pre_hook,
post_hook=_workflow_schedule_post_hook,
+ post_hook_fails_closed=True,
)
@@ -2537,6 +2562,20 @@ def _build_skyvern_mcp_overlays(
requires_browser=True,
pre_hook=_tab_close_pre_hook,
),
+ "list_workflow_runs": SchemaOverlay(
+ description=(
+ "List the saved runs of the workflow open in this chat, newest first. Copilot chat runs are "
+ "left out, and so are child runs unless include_child_runs=true. Filter by status, e.g. "
+ 'status=["terminated"]. Each run carries run_id, status, created_at and trigger_type. Read '
+ "one run's blocks and failure with get_run_results(workflow_run_id=<run_id>)."
+ ),
+ # search_key and error_code match stored run values, so matching run IDs would confirm guessed secrets.
+ hide_params=frozenset({"workflow_id", "search_key", "error_code"}),
+ binds_chat_workflow=True,
+ binds_chat_workflow_param="workflow_id",
+ post_hook=_workflow_run_list_post_hook,
+ post_hook_fails_closed=True,
+ ),
"list_workflow_schedules": _workflow_schedule_overlay(
"List the schedules of the workflow open in this chat, with each wfs_ ID. Check it before creating "
"or changing a schedule, and again afterwards to confirm.",
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 3c9131970..be1f259ab 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -333,6 +333,7 @@ RUN_BLOCKS_STAGNATION_WINDOW_SECONDS = 90
RUN_BLOCKS_POLL_INTERVAL_SECONDS = 5.0
COPILOT_SANDBOX_UNAVAILABLE_ERROR = "Sandboxed worker is unavailable; execution was not started."
+COPILOT_SNAPSHOT_PREPARATION_ERROR = "Unable to prepare the Copilot test-run snapshot; execution was not started."
# Block types that can reach exec() in the API process, so a run containing one may
# only proceed on the sandboxed worker. CODE compiles user code directly;
@@ -3724,14 +3725,14 @@ async def _run_blocks_and_collect_debug(
snapshot_provenance=snapshot.provenance,
exc_info=True,
)
- if dispatch_to_worker:
- return _copilot_sandbox_unavailable_result(
- organization_id=ctx.organization_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- )
return {
"ok": False,
- "error": "Unable to prepare the Copilot test-run snapshot; execution was not started.",
+ "error": COPILOT_SNAPSHOT_PREPARATION_ERROR,
+ "data": {
+ "workflow_run_id": None,
+ "failure_reason": COPILOT_SNAPSHOT_PREPARATION_ERROR,
+ "blocks": [],
+ },
}
if dispatch_workflow is not None:
@@ -5504,6 +5505,7 @@ def _record_run_blocks_result(
# the unattended page-observation self-heal verifier remains a separate lane.
copilot_ctx.completion_verification_result = None
copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else None
+ copilot_ctx.last_test_run_started = bool(run_id) or (isinstance(data, dict) and bool(data.get("blocks")))
run_browser_session_id = data.get("browser_session_id") if isinstance(data, dict) else None
copilot_ctx.last_run_blocks_browser_session_id = (
run_browser_session_id if isinstance(run_browser_session_id, str) and run_browser_session_id else None
diff --git a/skyvern/forge/sdk/routes/browser_sessions.py b/skyvern/forge/sdk/routes/browser_sessions.py
index c33b622e0..2165de6e5 100644
--- a/skyvern/forge/sdk/routes/browser_sessions.py
+++ b/skyvern/forge/sdk/routes/browser_sessions.py
@@ -493,6 +493,7 @@ async def get_browser_session(
app.STORAGE,
fail_download_lookup=True,
include_stream_transport=True,
+ concurrent_listings=True,
)
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 3022de41c..6d57c2c49 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -1728,6 +1728,9 @@ async def _finalise_normal_turn(
getattr(agent_result, "clear_persisted_completion_contract", False)
),
)
+ marker = _STAGED_COMMIT_LANDED.get()
+ if marker is not None:
+ marker[0] = True
except Exception:
# Undo any mid-turn degraded write so a failed commit fails the turn
# atomically instead of leaving canonical on a partial intermediate.
@@ -1749,6 +1752,10 @@ async def _finalise_normal_turn(
# candidate's token.
updated_workflow = agent_result.updated_workflow
proposal_disposition = _proposal_disposition(agent_result)
+ # Deliberately not the commit latch: this asks whether the change was applied by ANY means, and
+ # an ordinary auto-accept turn applies directly with has_staged_proposal False. It can diverge
+ # from the commit above when a candidate supersedes mid-commit, which is a receipt question of
+ # its own rather than this latch's.
workflow_applied = _effective_auto_accept(chat.auto_accept, agent_result)
narrative_payload = _with_terminal_narrative_metadata(
agent_result.narrative_payload,
@@ -1941,6 +1948,21 @@ _CANONICAL_ROLLED_BACK: contextvars.ContextVar[list[bool] | None] = contextvars.
default=None,
)
+# Whether this turn's staged commit actually reached canonical. ``asyncio.shield`` copies the
+# context, so a ``ContextVar.set`` inside the finalizer is invisible to the handler cleaning up
+# after it raises; mutating a list both contexts already hold is what survives that copy.
+_STAGED_COMMIT_LANDED: contextvars.ContextVar[list[bool] | None] = contextvars.ContextVar(
+ "copilot_staged_commit_landed",
+ default=None,
+)
+
+
+def _staged_commit_landed() -> bool:
+ """Did this turn's staged commit overwrite canonical? Answers the question the error paths
+ used to infer from ``auto_accept``, which is the setting, not the event."""
+ marker = _STAGED_COMMIT_LANDED.get()
+ return bool(marker and marker[0])
+
async def _restore_workflow_definition(original_workflow: Workflow | None, organization_id: str) -> None:
"""Roll the workflow back to ``original_workflow``.
@@ -2228,6 +2250,10 @@ async def _new_copilot_chat_post(
# than that some finalizer tried one.
canonical_rolled_back: list[bool] = [False]
_CANONICAL_ROLLED_BACK.set(canonical_rolled_back)
+ # Set by the finalizer once ``_commit_staged_workflow`` returns, so it means the commit
+ # landed rather than that auto-accept was on when the turn started.
+ staged_commit_landed: list[bool] = [False]
+ _STAGED_COMMIT_LANDED.set(staged_commit_landed)
cancel_watcher: asyncio.Task[None] | None = None
current_code_available = False
turn_index = 0
@@ -2305,11 +2331,7 @@ async def _new_copilot_chat_post(
# finalizer's alone, and a staged turn leaves ``workflow_was_persisted`` false even
# after committing, so it has to be read separately or the reply denies a change the
# user can see.
- committed_staged = (
- finalise_started
- and chat is not None
- and _should_commit_staged_workflow(chat.auto_accept, agent_result)
- )
+ committed_staged = _staged_commit_landed()
rolled_back = canonical_rolled_back[0]
persisted_directly = (
agent_result is not None and agent_result.workflow_was_persisted and not rolled_back
@@ -2340,7 +2362,12 @@ async def _new_copilot_chat_post(
)
restore_failed = True
if chat is not None:
- workflow_modified = agent_result is not None and agent_result.workflow_was_persisted and not restored
+ # A landed staged commit modified the workflow even though it leaves
+ # workflow_was_persisted False; the receipt below and the stale-card clear both
+ # depend on that.
+ workflow_modified = (
+ agent_result is not None and agent_result.workflow_was_persisted and not restored
+ ) or _staged_commit_landed()
# Pre-bake restored here: the recovered AgentResult has workflow_was_persisted=False,
# so _persist_proposed_workflow_state's own restored check would always read False.
recovered_result, failure = _build_recoverable_route_agent_result(
@@ -2353,9 +2380,6 @@ async def _new_copilot_chat_post(
(restored and not chat_request.keep_pending_proposal)
or workflow_modified
or getattr(agent_result, "clear_proposed_workflow", False)
- # A staged commit that already succeeded before this exception fired
- # still invalidates a stale kept proposal, same as the non-recovery path.
- or _should_commit_staged_workflow(chat.auto_accept, agent_result)
# A failed rollback leaves canonical's true state unverified — don't
# honor keep_pending_proposal against an assumption that didn't hold.
or restore_failed
diff --git a/skyvern/forge/sdk/services/org_auth_service.py b/skyvern/forge/sdk/services/org_auth_service.py
index e445f947c..202ef253c 100644
--- a/skyvern/forge/sdk/services/org_auth_service.py
+++ b/skyvern/forge/sdk/services/org_auth_service.py
@@ -21,6 +21,7 @@ from skyvern.forge.sdk.core.organization_age_cache import remember_organization_
from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.models import TokenPayload
from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken, OrganizationAuthTokenType
+from skyvern.forge.sdk.services.request_principal import resolve_request_principal
from skyvern.forge.sdk.workflow.models.tags import CallerType
LOG = structlog.get_logger()
@@ -178,6 +179,7 @@ async def get_current_org(
app.DATABASE,
user_agent=user_agent,
fern_language=x_fern_language,
+ authorization=authorization,
)
elif authorization:
organization = await authenticate_helper(
@@ -279,6 +281,11 @@ async def _get_current_org_for_token_types(
fern_language=fern_language,
)
apply_request_org_context(validation.organization)
+ await resolve_request_principal(
+ validation.organization.organization_id,
+ api_key_type=_api_key_type(_decode_token_before_cache(x_api_key)),
+ bearer_token=_extract_bearer_token(authorization),
+ )
return validation.organization
@@ -349,13 +356,15 @@ async def get_current_org_with_authentication(
)
-def _extract_bearer_token(authorization: str) -> str | None:
+def _extract_bearer_token(authorization: str | None) -> str | None:
"""Return the bearer token, or None for any other scheme.
Self-hosted deployments commonly gate the UI origin with HTTP basic auth, so the browser
attaches ``Authorization: Basic ...`` to same-origin API calls too. Treating that as a
Skyvern token rejects the request with an auth-method error that hides the real cause.
"""
+ if not authorization:
+ return None
scheme, separator, token = authorization.partition(" ")
if not separator or scheme.lower() != "bearer":
return None
@@ -366,6 +375,15 @@ def _extract_bearer_token(authorization: str) -> str | None:
async def authenticate_helper(
authorization: str,
attribution_header: str | None = None,
+) -> Organization:
+ organization = await _authenticate_bearer_organization(authorization, attribution_header)
+ await resolve_request_principal(organization.organization_id, bearer_token=_extract_bearer_token(authorization))
+ return organization
+
+
+async def _authenticate_bearer_organization(
+ authorization: str,
+ attribution_header: str | None = None,
) -> Organization:
token = _extract_bearer_token(authorization)
if not token:
@@ -408,6 +426,7 @@ async def get_current_user_id(
app.DATABASE,
user_agent=user_agent,
fern_language=x_fern_language,
+ authorization=authorization,
)
if organization:
return f"{organization.organization_id}_user"
@@ -441,6 +460,7 @@ async def get_current_user_id_or_none(
app.DATABASE,
user_agent=user_agent,
fern_language=x_fern_language,
+ authorization=authorization,
)
is_member = await app.AGENT_FUNCTION.validate_user_organization_membership(
user_id=user_id,
@@ -727,12 +747,18 @@ def _claims_ui_session(payload: dict[str, object] | None) -> bool:
return payload is not None and payload.get("token_type") == OrganizationAuthTokenType.ui_session.value
+def _api_key_type(payload: dict[str, object] | None) -> OrganizationAuthTokenType:
+ # Safe once the key has authenticated: a ui_session row is only accepted with this signed claim.
+ return OrganizationAuthTokenType.ui_session if _claims_ui_session(payload) else OrganizationAuthTokenType.api
+
+
async def get_current_org_cached(
x_api_key: str,
db: AgentDB,
*,
user_agent: str | None = None,
fern_language: str | None = None,
+ authorization: str | None = None,
) -> Organization:
payload = _decode_token_before_cache(x_api_key)
_validate_token_expiry_before_cache(payload)
@@ -753,6 +779,11 @@ async def get_current_org_cached(
fern_language=fern_language,
)
apply_request_org_context(organization)
+ await resolve_request_principal(
+ organization.organization_id,
+ api_key_type=_api_key_type(payload),
+ bearer_token=_extract_bearer_token(authorization),
+ )
return organization
@@ -794,6 +825,7 @@ async def get_current_caller_context(
app.DATABASE,
user_agent=user_agent,
fern_language=x_fern_language,
+ authorization=authorization,
)
apply_request_org_context(organization)
# x-user-agent is spoofable and is NOT an access-control check —
@@ -818,9 +850,11 @@ async def get_current_caller_context(
# concurrently so JWT validation cost is paid in parallel, not serially.
user_id, organization = await asyncio.gather(
authenticate_user_helper(authorization),
- authenticate_helper(authorization),
+ _authenticate_bearer_organization(authorization),
)
apply_request_org_context(organization)
+ # Resolved here, not in the gathered task: a context write there would not reach this task.
+ await resolve_request_principal(organization.organization_id, bearer_token=_extract_bearer_token(authorization))
return CallerContext(
organization=organization,
caller_id=user_id,
diff --git a/skyvern/forge/sdk/services/request_principal.py b/skyvern/forge/sdk/services/request_principal.py
new file mode 100644
index 000000000..5e45510aa
--- /dev/null
+++ b/skyvern/forge/sdk/services/request_principal.py
@@ -0,0 +1,105 @@
+import asyncio
+from contextvars import ContextVar
+from dataclasses import dataclass
+from enum import StrEnum
+
+import structlog
+from fastapi import HTTPException
+
+from skyvern.forge import app
+from skyvern.forge.request_logging import set_request_principal
+from skyvern.forge.sdk.schemas.organizations import OrganizationAuthTokenType
+
+LOG = structlog.get_logger()
+
+# ponytail: a failing identity lookup is retried by every request that carries a bearer, each bounded by
+# this timeout. Cache the failure briefly if an identity-provider outage ever shows up as request latency.
+BEARER_IDENTITY_TIMEOUT_SECONDS = 2.0
+
+
+class AuthKind(StrEnum):
+ api_key = "api_key"
+ ui_session = "ui_session"
+ bearer = "bearer"
+ # The key authorized the request; a valid bearer for the same organization supplied the user.
+ api_key_and_bearer = "api_key_and_bearer"
+ ui_session_and_bearer = "ui_session_and_bearer"
+
+
+@dataclass(frozen=True)
+class BearerIdentity:
+ user_id: str
+ # As found in the token, never normalized: the claim shapes spell one role differently.
+ org_role: str | None = None
+ org_role_claim: str | None = None
+
+
+@dataclass(frozen=True)
+class RequestPrincipal:
+ organization_id: str
+ auth_kind: AuthKind
+ user_id: str | None = None
+ org_role: str | None = None
+ org_role_claim: str | None = None
+
+
+_ResolutionKey = tuple[str, OrganizationAuthTokenType | None, str | None]
+_request_principal: ContextVar[tuple[_ResolutionKey, RequestPrincipal] | None] = ContextVar(
+ "request_principal", default=None
+)
+
+
+def get_request_principal() -> RequestPrincipal | None:
+ resolved = _request_principal.get()
+ return resolved[1] if resolved else None
+
+
+async def _resolve_bearer_identity(bearer_token: str, organization_id: str) -> BearerIdentity | None:
+ try:
+ identity = await asyncio.wait_for(
+ app.AGENT_FUNCTION.resolve_bearer_identity(bearer_token, organization_id),
+ timeout=BEARER_IDENTITY_TIMEOUT_SECONDS,
+ )
+ except HTTPException:
+ return None
+ except Exception:
+ LOG.warning("Failed to resolve the bearer identity for the request principal", exc_info=True)
+ return None
+ return identity if isinstance(identity, BearerIdentity) else None
+
+
+def _auth_kind(api_key_type: OrganizationAuthTokenType | None, has_user: bool) -> AuthKind:
+ if api_key_type is None:
+ return AuthKind.bearer
+ if api_key_type == OrganizationAuthTokenType.ui_session:
+ return AuthKind.ui_session_and_bearer if has_user else AuthKind.ui_session
+ return AuthKind.api_key_and_bearer if has_user else AuthKind.api_key
+
+
+async def resolve_request_principal(
+ organization_id: str,
+ *,
+ api_key_type: OrganizationAuthTokenType | None = None,
+ bearer_token: str | None = None,
+) -> RequestPrincipal:
+ """Record who the authenticated request acts as. Runs after authentication and never rejects.
+
+ ``api_key_type`` is the key that authorized the request, or None when the bearer did. The user
+ and role only ever come from a bearer the deployment's identity provider vouches for.
+ """
+ resolution_key = (organization_id, api_key_type, bearer_token)
+ resolved = _request_principal.get()
+ if resolved is not None and resolved[0] == resolution_key:
+ return resolved[1]
+
+ identity = await _resolve_bearer_identity(bearer_token, organization_id) if bearer_token else None
+ principal = RequestPrincipal(
+ organization_id=organization_id,
+ auth_kind=_auth_kind(api_key_type, identity is not None),
+ user_id=identity.user_id if identity else None,
+ org_role=identity.org_role if identity else None,
+ org_role_claim=identity.org_role_claim if identity else None,
+ )
+ _request_principal.set((resolution_key, principal))
+ set_request_principal(principal)
+ return principal
diff --git a/skyvern/forge/sdk/workflow/context_manager.py b/skyvern/forge/sdk/workflow/context_manager.py
index 826b275f6..cc8d0bdbf 100644
--- a/skyvern/forge/sdk/workflow/context_manager.py
+++ b/skyvern/forge/sdk/workflow/context_manager.py
@@ -1,7 +1,7 @@
import asyncio
import copy
import re
-from collections.abc import Awaitable, Callable, Iterable, Mapping
+from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from contextlib import suppress
from dataclasses import dataclass
from datetime import datetime, timezone
@@ -71,6 +71,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
WorkflowParameterType,
)
from skyvern.forge.sdk.workflow.page_derived_templates import RootClass, classify_roots
+from skyvern.schemas.workflows import BlockStatus
from skyvern.utils.phone_validation import looks_like_phone_identifier, normalize_identifier
from skyvern.utils.secret_redaction import collect_redactable_secret_values, is_redactable_secret_value
from skyvern.utils.strings import generate_random_string
@@ -97,6 +98,18 @@ def _normalize_credential_totp_identifier(value: object) -> str | None:
BlockMetadata = dict[str, str | int | float | bool | dict | list | None]
BitwardenCredentials = tuple[str | None, str | None, str | None, str | None]
+BLOCK_OUTCOME_FAILURE_REASON_MAX_CHARS = 2000
+
+
+@dataclass(frozen=True)
+class BlockOutcome:
+ """How a block ended, in one shape for every block type. A block that never ran has no record."""
+
+ status: BlockStatus
+ error_codes: list[str]
+ failure_reason: str | None
+
+
jinja_sandbox_env = SandboxedEnvironment()
RANDOM_SECRET_ID_PREFIX = "placeholder_"
@@ -322,6 +335,8 @@ class WorkflowRunContext:
# workflow definition in two places (SKY-9147).
self._block_workflow_system_prompts: dict[str, str | None] = {}
self.blocks_metadata: dict[str, BlockMetadata] = {}
+ # Kept apart from values and blocks_metadata so no template or branch-evaluation snapshot can see it.
+ self.block_outcomes: dict[str, BlockOutcome] = {}
self.parameters: dict[str, PARAMETER_TYPE] = {}
self.values: dict[str, Any] = {}
self.secrets: dict[str, Any] = {}
@@ -503,6 +518,19 @@ class WorkflowRunContext:
label = ""
return self.blocks_metadata.get(label, BlockMetadata())
+ def record_block_outcome(
+ self, label: str, status: BlockStatus, error_codes: Sequence[str], failure_reason: str | None
+ ) -> None:
+ if failure_reason is not None:
+ # Mask before cutting: a cut can leave the head of a secret in the stored reason.
+ failure_reason = str(self.mask_secrets_in_data(failure_reason))[:BLOCK_OUTCOME_FAILURE_REASON_MAX_CHARS]
+ self.block_outcomes[label] = BlockOutcome(
+ status=status, error_codes=list(error_codes), failure_reason=failure_reason
+ )
+
+ def get_block_outcome(self, label: str) -> BlockOutcome | None:
+ return self.block_outcomes.get(label)
+
def record_block_workflow_system_prompt(self, label: str, value: str | None) -> None:
"""Record the effective ``workflow_system_prompt`` a block resolved to.
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 6a708acfe..e1adfceb4 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -159,7 +159,6 @@ from skyvern.forge.sdk.api.llm.schema_validator import (
)
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.artifact.storage.base import get_download_retry_started_at, is_file_from_retry_attempt
-from skyvern.forge.sdk.copilot.block_goal_wrapping import compose_mini_goal
from skyvern.forge.sdk.copilot.code_block_security import INERT_SLOT_NAME
from skyvern.forge.sdk.copilot.code_block_steps import analyze_code_actions
from skyvern.forge.sdk.copilot.reached_download_target import (
@@ -288,6 +287,7 @@ from skyvern.forge.sdk.workflow.secret_encryption import (
is_encrypted_secret,
is_full_template_reference,
)
+from skyvern.forge.taskv3.goal_composition import CodeProgressRecord
from skyvern.forge.taskv3.handoff_redaction import pin_caller_authored_block_urls
from skyvern.schemas.browser_session_close import BrowserSessionCloseReason
from skyvern.schemas.emails import EmailBodyFormat
@@ -1671,6 +1671,43 @@ class Block(BaseModel, abc.ABC):
current_value: str | None = None,
current_index: int | None = None,
**kwargs: dict,
+ ) -> BlockResult:
+ result = await self._execute_to_block_result(
+ workflow_run_id,
+ parent_workflow_run_block_id,
+ organization_id,
+ browser_session_id,
+ current_value,
+ current_index,
+ **kwargs,
+ )
+ self.record_result_outcome(workflow_run_id, result)
+ return result
+
+ def record_result_outcome(self, workflow_run_id: str, result: BlockResult) -> None:
+ if result.status is None:
+ return
+ try:
+ workflow_run_context = self.get_workflow_run_context(workflow_run_id)
+ except WorkflowRunContextNotInitialized:
+ # The run was torn down while this block finished; nothing is left to read the record.
+ return
+ try:
+ workflow_run_context.record_block_outcome(
+ self.label, result.status, result.error_codes, result.failure_reason
+ )
+ except Exception:
+ LOG.warning("Failed to record block outcome", block_label=self.label, exc_info=True)
+
+ async def _execute_to_block_result(
+ self,
+ workflow_run_id: str,
+ parent_workflow_run_block_id: str | None = None,
+ organization_id: str | None = None,
+ browser_session_id: str | None = None,
+ current_value: str | None = None,
+ current_index: int | None = None,
+ **kwargs: dict,
) -> BlockResult:
# block_type slices the 303s p95 by block kind — task/for_loop/code/extraction
# have wildly different latency profiles. Set early so it's present even if
@@ -7493,25 +7530,33 @@ async def wrapper({default_args}):
return idx
return None
- def _compose_heal_goal(self, *, workflow_run_context: WorkflowRunContext, failing_line: int | None) -> str:
- safe_main = workflow_run_context.mask_secrets_in_data(self.prompt or "")
- # Steps are a code-derived outline, not an authored goal, so they may only narrow one.
- if not self.prompt:
- return safe_main
- matched_step = self._match_step_for_failing_line(failing_line) if failing_line is not None else None
- if matched_step is None or not matched_step.description:
- return safe_main
+ def _compose_heal_goal(self, *, workflow_run_context: WorkflowRunContext) -> str:
+ return workflow_run_context.mask_secrets_in_data(self.prompt or "")
+
+ def _code_progress_record(
+ self, *, workflow_run_context: WorkflowRunContext, failing_line: int | None
+ ) -> CodeProgressRecord | None:
+ if failing_line is None:
+ return None
+ index = self._matched_step_index_for_failing_line(failing_line)
+ if index is None:
+ return None
steps = self.steps or []
- matched_index = next((idx for idx, step in enumerate(steps) if step is matched_step), None)
- if matched_index is None:
- matched_index = len(steps) - 1
- descriptions = [matched_step.description] + [
- step.description for step in steps[matched_index + 1 :] if step.description
- ]
- safe_mini = "\nThen: ".join(
- workflow_run_context.mask_secrets_in_data(description) for description in descriptions
+ failed = steps[index]
+ if not failed.description:
+ return None
+
+ def masked(chunk: list[CodeBlockStep]) -> tuple[str, ...]:
+ return tuple(
+ workflow_run_context.mask_secrets_in_data(step.description) for step in chunk if step.description
+ )
+
+ return CodeProgressRecord(
+ before=masked(steps[:index]),
+ failed_step=workflow_run_context.mask_secrets_in_data(failed.description),
+ failed_line=failing_line,
+ after=masked(steps[index + 1 :]),
)
- return compose_mini_goal(main_goal=safe_main, mini_goal=safe_mini)
async def _record_unregistered_download_intent(
self,
@@ -7706,10 +7751,7 @@ async def wrapper({default_args}):
escalation_step: Step | None = None
recovery_block_id: str | None = None
try:
- navigation_goal = self._compose_heal_goal(
- workflow_run_context=workflow_run_context,
- failing_line=failing_line,
- )
+ navigation_goal = self._compose_heal_goal(workflow_run_context=workflow_run_context)
navigation_payload = {
parameter.key: workflow_run_context.get_value_or_none(parameter.key) for parameter in self.parameters
}
@@ -7899,6 +7941,9 @@ async def wrapper({default_args}):
workflow_owned_recovery=True,
recovery_credential_parameter_keys=login_credential_parameter_keys,
recovery_release_parameter_keys=recovery_release_parameter_keys,
+ recovery_code_progress=self._code_progress_record(
+ workflow_run_context=workflow_run_context, failing_line=failing_line
+ ),
)
finally:
current_context.task_id = previous_task_id
@@ -16860,6 +16905,9 @@ def _neutralize_jinja_delimiters(value: Any) -> Any:
return value
+CONDITIONAL_DEBUG_OUTPUT_FIELDS = frozenset({"llm_prompt", "llm_response", "evaluations"})
+
+
class BranchEvaluationContext:
"""Collection of runtime data that BranchCriteria evaluators can consume."""
@@ -16903,21 +16951,52 @@ class BranchEvaluationContext:
"workflow_run_id",
}
+ # Block references registered as a copy of `<label>_output`, this run's and carried in.
+ block_aliases = ctx.workflow_run_outputs.keys() | ctx.carried_block_labels
+ workflow_definition = ctx.workflow.workflow_definition if ctx.workflow else None
+ conditional_output_keys: set[str] = set()
+ for block in get_all_blocks(workflow_definition.blocks if workflow_definition else []):
+ if block.block_type != BlockType.CONDITIONAL:
+ continue
+ conditional_output_keys.add(block.output_parameter.key)
+ # A parameter may share the label; only a registered alias is the block's own output.
+ alias = block.output_parameter.key.removesuffix("_output")
+ if alias in block_aliases:
+ conditional_output_keys.add(alias)
+
snapshot: dict[str, Any] = {}
for key, value in raw_values.items():
# Skip noisy keys
if key in keys_to_skip:
continue
+ # A synthetic branch-evaluation block records its raw response under a generated label; a key the
+ # author declared, or a block produced, under that prefix is theirs and stays.
+ if key.startswith("prompt_branch_eval_") and key not in ctx.parameters and key not in block_aliases:
+ continue
+
# For block outputs (dicts with extracted_information), only include extracted_information
if isinstance(value, dict) and "extracted_information" in value:
extracted = value.get("extracted_information")
if extracted is not None:
snapshot[key] = extracted
+ elif key in conditional_output_keys and isinstance(value, dict):
+ # An earlier conditional's stored prompt embedded this same snapshot; re-sending it, its raw
+ # response and its per-branch detail grows every later evaluation. The routing fields stay.
+ snapshot[key] = {
+ field: item for field, item in value.items() if field not in CONDITIONAL_DEBUG_OUTPUT_FIELDS
+ }
else:
# Include parameter values directly
snapshot[key] = value
+ # `<label>` is registered as a copy of `<label>_output`. Keep one; a copy that differs (keys merged
+ # across loop iterations) holds something the output does not, so it stays.
+ for label in block_aliases:
+ output_key = f"{label}_output"
+ if label in snapshot and output_key in snapshot and snapshot[label] == snapshot[output_key]:
+ del snapshot[label]
+
# Copy loop variables (current_value, current_index, current_item) to top level
# Required for pure NatLang expressions like "current_value['date']" to work
if self.block_label:
@@ -17973,12 +18052,14 @@ class ConditionalBlock(Block):
"""
Evaluate natural language branch conditions in batch.
- All prompt-based conditions are batched into ONE LLM call for performance.
+ Prompt-based conditions are batched into a synthetic ExtractionBlock per evaluation attempt.
Jinja parts ({{ }}) are pre-rendered before sending to LLM.
Evaluation strategy:
- - If any condition is pure natural language, use ExtractionBlock for browser/page context.
- - If all conditions contain Jinja and are pre-rendered, use direct LLM call (no browser context).
+ - Every batch runs through the ExtractionBlock with the run's browser session, including
+ batches whose conditions are all pre-rendered Jinja.
+ - The workflow context snapshot is added to the prompt only when a condition is pure natural
+ language or its Jinja fails to render.
Returns:
A tuple of (results, rendered_expressions, extraction_goal, llm_response):
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 49ac9a867..7d2cad764 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -8626,11 +8626,14 @@ class WorkflowService:
# This fires both when the block requires_agent (first run) and when
# cached code failed and agent fallback re-ran the conditional
# (fallback_episode_id is set when the script path failed).
+ # A block that routed despite a failed branch evaluation is still
+ # `completed`, but that branch has no result for the reviewer to learn.
if (
is_script_run
and (block_requires_agent or attempt.fallback_episode_id)
and workflow_run_block_result.status == BlockStatus.completed
and branch_metadata
+ and not branch_metadata.get("evaluation_error")
and is_adaptive_caching(workflow, workflow_run)
):
await self._record_conditional_agent_episode(
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index dd7a4d360..939744fde 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -111,10 +111,10 @@ MIN_ACTION_STEPS = 24
# at or below MIN_ACTION_STEPS keeps exactly DEFAULT_MAX_TOKENS; only larger budgets rise, so a long
# block's raised step cap isn't silently nullified by the flat token ceiling.
MAX_TOKENS_PER_ACTION_STEP = DEFAULT_MAX_TOKENS // MIN_ACTION_STEPS
-# The scaling clamps here: the token guard is a runaway backstop, not a budget, and a caller's step
-# cap is not bounded at the route layer, so an extreme value must not carry the ceiling away with
-# it. 4x covers every observed legitimate long-block need (~2x) with margin. Deliberately asymmetric:
-# turns/tool-calls scale unbounded (they cost loop iterations), tokens are the direct-spend guard.
+# The scaling clamps here: tokens are the direct-spend guard, and a caller's step cap is not bounded at the
+# route layer, so an extreme value must not carry the ceiling away with it. Because the transcript grows each
+# turn, the per-step sizing can bind before the step cap on long forms; the loop's progress-gated token grant
+# may then raise it, never past this ceiling. Turns/tool-calls scale unbounded (they cost loop iterations).
MAX_TOKENS_CEILING = 4 * DEFAULT_MAX_TOKENS
# Left between the judge's timeout and the run's deadline, so a judge call cannot be what ends the run.
GOAL_CHECK_DEADLINE_MARGIN_SECONDS = 2.0
@@ -179,8 +179,9 @@ This task cannot finish as completed until a file download has finished. Trigger
def taskv3_runaway_backstops(max_action_steps: int | None) -> tuple[int, int, int]:
"""Return (max_turns, max_tool_calls, max_tokens) anti-runaway guards for an action-step budget.
- Generous enough that a productive run is bounded by max_action_steps, not by these guards; with
- no action-step budget, fall back to the engine's fixed defaults."""
+ Turns and tool calls are generous enough that a productive run is bounded by max_action_steps; the token
+ guard can bind first on a long form, where the loop's progress-gated token grant applies. With no
+ action-step budget, fall back to the engine's fixed defaults."""
if not max_action_steps:
return DEFAULT_MAX_TURNS, DEFAULT_MAX_TOOL_CALLS, DEFAULT_MAX_TOKENS
return (
diff --git a/skyvern/forge/taskv3/goal_composition.py b/skyvern/forge/taskv3/goal_composition.py
index 6afaaa431..bad66a83f 100644
--- a/skyvern/forge/taskv3/goal_composition.py
+++ b/skyvern/forge/taskv3/goal_composition.py
@@ -88,6 +88,26 @@ def present_page_derived(field: str, row_value: str, render: PageDerivedRender |
return PresentedField(text=qualifier + row_value, presentation=presentation, reason=reason)
+@dataclass(frozen=True)
+class CodeProgressRecord:
+ """A failed code block's step outline split at the step whose lines raised.
+
+ Steps are in source order, so a loop or branch means ``before`` is not proof those steps ran.
+ """
+
+ before: tuple[str, ...]
+ failed_step: str
+ failed_line: int
+ after: tuple[str, ...]
+
+
+def render_code_progress_section(record: CodeProgressRecord) -> str:
+ lines = [f"- Earlier in the code: {step}" for step in record.before]
+ lines.append(f"- Raised an error at line {record.failed_line}: {record.failed_step}")
+ lines.extend(f"- Later in the code: {step}" for step in record.after)
+ return "Code outline (a record of this block's code in source order, not steps to perform):\n" + "\n".join(lines)
+
+
@dataclass(frozen=True)
class GoalDirectives:
"""Everything that shapes a Task V3 block's goal beyond the navigation goal itself.
@@ -110,13 +130,14 @@ class GoalDirectives:
block_context_section: str = ""
# Set when a field above carries a ⟦"…"⟧ page-value span.
page_data_note: bool = False
+ code_progress: CodeProgressRecord | None = None
def compose_goal(navigation_goal: str, directives: GoalDirectives) -> str:
"""Build the goal the model is given, appending each directive in a fixed order.
Order is part of the contract: the model reads the extraction instruction before the schema it
- must conform to, and the framing and block-context sections land last so run-shaped context
+ must conform to, and the framing, block-context and code-outline sections land last so run-shaped context
never separates a criterion from the goal it qualifies.
"""
goal = navigation_goal
@@ -171,6 +192,8 @@ def compose_goal(navigation_goal: str, directives: GoalDirectives) -> str:
goal = f"{goal}\n\n{directives.framing}".strip()
if directives.block_context_section:
goal = f"{goal}\n\n{directives.block_context_section}".strip()
+ if directives.code_progress is not None:
+ goal = f"{goal}\n\n{render_code_progress_section(directives.code_progress)}".strip()
return goal
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index 1d8899b6a..a4fbc657a 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -722,6 +722,11 @@ UNCHARGED_REFUSAL_GRACE = ACTION_LOOP_NUDGE_AFTER
# precision is measurable on the canary; change only with the dashboards that read them.
ACTION_BUDGET_EXTENDED_EVENT = "taskv3 loop action budget extended"
ACTION_BUDGET_EXTENSION_REFUSED_EVENT = "taskv3 loop action budget extension refused"
+# The token backstop trips first on long forms (each turn re-sends the transcript), so the gate may raise it alone.
+# What binds is page-change evidence after a never-made billable action, plus the 3x/pool/ceiling bound.
+TOKEN_BUDGET_EXTENDED_EVENT = "taskv3 loop token budget extended"
+TOKEN_BUDGET_EXTENSION_REFUSED_EVENT = "taskv3 loop token budget extension refused"
+TOKEN_EXTENSION_FUNDED_ROUNDS = 2
BATCH_SKIP_TOGGLE_EXEMPT_EVENT = "taskv3 loop batch skip exempted toggle click"
CREDENTIAL_RESUBMIT_REFUSED_EVENT = "taskv3 loop credential resubmit refused"
EXTRACTION_ENTRY_REFUSED_EVENT = "taskv3 loop extraction entry refused"
@@ -3112,7 +3117,7 @@ class LoopState:
# The action round of the latest positive page-change evidence (SKY-15264, SKY-15666); the
# budget-extension gate reads it, so what counts as evidence is load-bearing, not cosmetic.
last_change_evidence_step: int | None = None
- # Budget caps. The four are re-derived and applied ATOMICALLY on an extension grant: a partial
+ # Budget caps. The four are re-derived and applied ATOMICALLY on a step-budget grant: a partial
# update converts a step-cap death into a token-cap death, so they move together or not at all.
max_turns: int
max_tool_calls: int
@@ -3132,6 +3137,20 @@ class LoopState:
# stays linear. The evidence input is last_change_evidence_step above.
budget_extensions_granted: int = 0
original_action_steps: int | None = None
+ # The step cap the token guard is currently sized for; a token-only grant moves it past max_action_steps.
+ token_extension_basis: int = 0
+ # Evidence recency is counted in action rounds, which a turn without an action does not advance, so a
+ # token grant instead spends the evidence it was granted on and the next grant needs new evidence.
+ token_extension_evidence_spent: bool = False
+ token_extensions_granted: int = 0
+ token_grant_action_steps: int | None = None
+ # Billable (tool, args) keys that have succeeded, and the dispatch of the first never-seen one since the last
+ # token grant: a repeated action (a hover, an identical submit retry) does not re-arm a token grant. Ordered
+ # against evidence by a per-event sequence, since a batch's action and its evidence share one step.
+ succeeded_action_keys: set[tuple[str, str]] = field(default_factory=set)
+ event_seq: int = 0
+ novel_action_seq: int | None = None
+ last_change_evidence_seq: int = 0
token_clamp_reported: bool = False
# Final-turn grant (budget-exhaustion final turn): mirrors budget_extension_granted's shape. A
# budget-cap trip anywhere in the loop sets this and buys one more unconstrained model turn. One
@@ -3279,6 +3298,9 @@ async def run_agent_tool_loop(
def _note_page_change_evidence() -> None:
st.last_change_evidence_step = st.action_steps
+ st.token_extension_evidence_spent = False
+ st.event_seq += 1
+ st.last_change_evidence_seq = st.event_seq
async def _consume_refresh_signal(
ctx: SkyvernContext, tool_name: str, remaining: list[Any], round_actions: list[Any], *, drop: bool
@@ -3608,6 +3630,103 @@ async def run_agent_tool_loop(
started_at = time.monotonic()
deadline_at = started_at + deadline_seconds if deadline_seconds is not None else None
+ def _extension_gate(extension: int, headroom_gain: tuple[int, int, int]) -> tuple[bool, str]:
+ refresh_ctx = skyvern_context.current()
+ if refresh_ctx is not None and refresh_ctx.refresh_working_page:
+ # A pending refresh voids the next action and re-baselines the page: a grant must not race
+ # it and spend the extension on pre-reload evidence.
+ return False, "refresh_pending"
+ return _budget_extension_gate(
+ st.action_steps,
+ st.last_change_evidence_step,
+ st.action_warned,
+ # CURRENT confirmed stalled-ness by the ledger's own rules: form_armed (the latest look
+ # showed a form) plus a window of fruitless actions. Not the one-shot telemetry latch, and
+ # never a bare counter — a form-less page increments the counter but must not be judged
+ # stuck by it.
+ st.progress is not None
+ and st.progress.form_armed
+ and st.progress.actions_since_progress >= st.progress.window,
+ activity,
+ deadline_at,
+ extension,
+ seconds_per_step=(time.monotonic() - started_at) / max(st.action_steps, 1),
+ headroom_gain=headroom_gain,
+ )
+
+ def _token_budget_extended() -> bool:
+ if (
+ backstops_for_cap is None
+ or st.max_tokens is None
+ or st.max_action_steps is None
+ or st.action_steps >= st.max_action_steps
+ ):
+ return False
+ base_cap = st.original_action_steps or st.max_action_steps
+ basis = max(st.token_extension_basis, st.max_action_steps)
+ extension_limit = base_cap * ACTION_BUDGET_EXTENSION_MAX_FACTOR
+ if max_action_steps_ceiling is not None:
+ extension_limit = min(extension_limit, max_action_steps_ceiling)
+ extension = min(base_cap // 2, extension_limit - basis)
+ grant_max_tokens = st.max_tokens
+ if extension > 0:
+ grant_max_tokens = max(st.max_tokens, backstops_for_cap(basis + extension)[2])
+ if extension <= 0:
+ allowed, gate_reason = False, "extension_limit_reached"
+ elif st.total_tokens >= grant_max_tokens - final_turn_token_reserve:
+ allowed, gate_reason = False, "token_ceiling_reached"
+ elif st.token_extension_evidence_spent:
+ allowed, gate_reason = False, "no_recent_page_change_evidence"
+ elif st.token_grant_action_steps is not None and st.novel_action_seq is None:
+ # Perception alone re-earns evidence on a page whose content keeps changing (a clock, rotating
+ # text), so a renewal needs a new action, and page-change evidence stamped after it.
+ allowed, gate_reason = False, "no_new_action_since_last_token_grant"
+ elif st.token_grant_action_steps is not None and st.last_change_evidence_seq <= (st.novel_action_seq or 0):
+ allowed, gate_reason = False, "no_page_change_after_the_new_action"
+ else:
+ # Funding 2 rounds at the last turn's cost rarely binds here; the checks above and the gate's evidence
+ # check refuse a spiral, and its stall vetoes bind only on a form page or identical re-reads.
+ allowed, gate_reason = _extension_gate(
+ TOKEN_EXTENSION_FUNDED_ROUNDS, (0, 0, grant_max_tokens - st.max_tokens)
+ )
+ if not allowed:
+ LOG.info(
+ TOKEN_BUDGET_EXTENSION_REFUSED_EVENT,
+ guard="max_tokens",
+ gate_reason=gate_reason,
+ max_tokens=st.max_tokens,
+ total_tokens=st.total_tokens,
+ action_steps=st.action_steps,
+ max_action_steps=st.max_action_steps,
+ token_extensions_granted=st.token_extensions_granted,
+ base_cap=base_cap,
+ turn=st.turns,
+ )
+ return False
+ original_max_tokens = st.max_tokens
+ st.token_extensions_granted += 1
+ st.token_extension_basis = basis + extension
+ st.token_extension_evidence_spent = True
+ st.token_grant_action_steps = st.action_steps
+ st.novel_action_seq = None
+ st.max_tokens = grant_max_tokens
+ if activity is not None:
+ activity.tokens_remaining = st.max_tokens - st.total_tokens
+ LOG.info(
+ TOKEN_BUDGET_EXTENDED_EVENT,
+ guard="max_tokens",
+ original_max_tokens=original_max_tokens,
+ max_tokens=st.max_tokens,
+ token_extension_basis=st.token_extension_basis,
+ total_tokens=st.total_tokens,
+ action_steps=st.action_steps,
+ max_action_steps=st.max_action_steps,
+ token_extensions_granted=st.token_extensions_granted,
+ base_cap=base_cap,
+ turn=st.turns,
+ )
+ return True
+
# The task's starting URL is navigated during browser setup, before this loop runs, so a dead/removed
# starting posting never routes through the in-loop `navigate` tool — the model just observes the dead
# page and finishes (defaulting to failed). Classify that pre-loop navigation here so the dominant
@@ -3660,7 +3779,11 @@ async def run_agent_tool_loop(
if not st.final_turn_granted:
if deadline_seconds is not None and time.monotonic() - started_at > deadline_seconds:
top_of_turn_trip = f"deadline ({deadline_seconds:.0f}s) reached"
- elif st.max_tokens is not None and st.total_tokens >= max(0, st.max_tokens - final_turn_token_reserve):
+ elif (
+ st.max_tokens is not None
+ and st.total_tokens >= max(0, st.max_tokens - final_turn_token_reserve)
+ and not _token_budget_extended()
+ ):
top_of_turn_trip = f"max_tokens ({st.max_tokens}) reached"
elif st.turns >= st.max_turns:
top_of_turn_trip = f"max_turns ({st.max_turns}) reached"
@@ -4038,36 +4161,21 @@ async def run_agent_tool_loop(
# Reported from the grant branch below, never here — a refused extension leaves
# the cap where it was, so reporting on it would name a cap never in effect and
# would spend the once-per-run latch on a non-event.
- token_clamped = st.max_tokens is not None and token_gain == 0 and grant_max_turns > st.max_turns
- refresh_ctx = skyvern_context.current()
+ # Judged on the sizing function, not the live guard: a token-only grant may already have
+ # raised max_tokens to this cap's value, which is no ceiling.
+ token_clamped = (
+ st.max_tokens is not None
+ and next_tokens <= backstops_for_cap(st.max_action_steps)[2]
+ and grant_max_turns > st.max_turns
+ )
if st.max_action_steps >= extension_limit:
allowed, gate_reason = False, "extension_limit_reached"
elif raw_extension <= 0:
allowed, gate_reason = False, "cap_too_small"
elif extension <= 0:
allowed, gate_reason = False, "hard_step_ceiling"
- elif refresh_ctx is not None and refresh_ctx.refresh_working_page:
- # A pending refresh voids this very action and re-baselines the page: the grant
- # must not race it and spend the extension on pre-reload evidence.
- allowed, gate_reason = False, "refresh_pending"
else:
- allowed, gate_reason = _budget_extension_gate(
- st.action_steps,
- st.last_change_evidence_step,
- st.action_warned,
- # CURRENT confirmed stalled-ness by the ledger's own rules: form_armed
- # (the latest look showed a form) plus a window of fruitless actions. Not
- # the one-shot telemetry latch, and never a bare counter — a form-less page
- # increments the counter but must not be judged stuck by it.
- st.progress is not None
- and st.progress.form_armed
- and st.progress.actions_since_progress >= st.progress.window,
- activity,
- deadline_at,
- extension,
- seconds_per_step=(time.monotonic() - started_at) / max(st.action_steps, 1),
- headroom_gain=headroom_gain,
- )
+ allowed, gate_reason = _extension_gate(extension, headroom_gain)
# This read cannot see unabsorbed in-batch movement: action_steps is frozen during
# a batch, so the cap check trips on the batch's FIRST billable call — before any
# page action dispatches — and between-batch movement was absorbed when
@@ -4285,6 +4393,8 @@ async def run_agent_tool_loop(
if spec is not None and spec.touches_page and dispatch_ctx is not None and dispatch_ctx.task_id:
clear_task_nav_error_code(dispatch_ctx.task_id)
tool_started_at = time.monotonic()
+ st.event_seq += 1
+ dispatch_seq = st.event_seq
if spec is None:
result = ToolResult.error(f"unknown_tool: {tool_name}")
else:
@@ -4627,6 +4737,10 @@ async def run_agent_tool_loop(
_append_skipped_tool_results(st.messages, tool_calls[idx + 1 :], "perception stalled")
break
if spec is not None and spec.billable:
+ if result.status == "ok" and action_key not in st.succeeded_action_keys:
+ st.succeeded_action_keys.add(action_key)
+ if st.novel_action_seq is None:
+ st.novel_action_seq = dispatch_seq
if st.progress is not None:
st.progress.on_billable()
# Errored and refused dispatches count too: a repeat-failing action is the same
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 3a31e5776..f520b6dcf 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -933,6 +933,7 @@ async def _handle_script_termination(
label=cache_key,
failure_reason=str(e),
user_defined_errors=e.user_defined_errors,
+ error_codes=[error.error_code for error in e.user_defined_errors or []],
)
@@ -949,6 +950,7 @@ async def _update_workflow_block(
output: dict[str, Any] | list | str | None = None,
ai_fallback_triggered: bool | None = None,
user_defined_errors: list[UserDefinedError] | None = None,
+ error_codes: list[str] | None = None,
) -> None:
"""Update workflow_run_block status, optionally setting `script_run`.
@@ -1070,15 +1072,27 @@ async def _update_workflow_block(
# final_output is already set to `output` at line 596.
pass
- await app.DATABASE.observer.update_workflow_run_block(
+ updated_block = await app.DATABASE.observer.update_workflow_run_block(
workflow_run_block_id=workflow_run_block_id,
organization_id=context.organization_id if context else None,
status=status,
failure_reason=failure_reason,
output=final_output,
ai_fallback_triggered=ai_fallback_triggered,
+ error_codes=error_codes,
)
+ # The row carries the block label even when the caller passed none (a cached wait block).
+ if updated_block.label:
+ try:
+ app.WORKFLOW_CONTEXT_MANAGER.get_workflow_run_context(context.workflow_run_id).record_block_outcome(
+ updated_block.label, status, updated_block.error_codes or [], failure_reason
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to record cached block outcome", workflow_run_block_id=workflow_run_block_id, exc_info=True
+ )
+
recorded_output_parameter = await _record_output_parameter_value(
context.workflow_run_id,
context.workflow_id,
@@ -1557,8 +1571,8 @@ async def _fallback_to_ai_run(
# Update workflow block with failure reason (include detected errors if any)
task_failure_reason = str(error)
- if detected_errors:
- error_codes = [e.error_code for e in detected_errors]
+ error_codes = [e.error_code for e in detected_errors]
+ if error_codes:
task_failure_reason = f"{task_failure_reason}. Detected errors: {', '.join(error_codes)}"
if workflow_run_block_id:
@@ -1573,6 +1587,7 @@ async def _fallback_to_ai_run(
step_id=script_step_id,
step_status=StepStatus.failed,
label=cache_key,
+ error_codes=error_codes,
)
return
@@ -1751,6 +1766,11 @@ async def _fallback_to_ai_run(
task = refreshed_task
if task.status in [TaskStatus.terminated, TaskStatus.failed]:
failure_reason = task.failure_reason
+ error_codes = [
+ error["error_code"]
+ for error in task.errors or []
+ if isinstance(error, dict) and isinstance(error.get("error_code"), str)
+ ]
await _update_workflow_block(
workflow_run_block_id,
BlockStatus(task.status.value),
@@ -1758,6 +1778,7 @@ async def _fallback_to_ai_run(
failure_reason=failure_reason,
label=cache_key,
ai_fallback_triggered=True,
+ error_codes=error_codes,
)
# 5. After successful AI execution, regenerate the script block and create new version
@@ -4099,7 +4120,7 @@ async def loop(
await loop_block.record_output_parameter_value(workflow_run_context, workflow_run_id, [])
# step 4. build response (success/failure) given the complete_if_empty value
if complete_if_empty:
- await loop_block.build_block_result(
+ empty_result = await loop_block.build_block_result(
success=True,
failure_reason=None,
output_parameter_value=[],
@@ -4107,16 +4128,18 @@ async def loop(
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
)
- return
else:
- await loop_block.build_block_result(
+ empty_result = await loop_block.build_block_result(
success=False,
failure_reason="No iterable value found for the loop block",
status=BlockStatus.terminated,
workflow_run_block_id=workflow_run_block_id,
organization_id=organization_id,
)
+ loop_block.record_result_outcome(workflow_run_id, empty_result)
+ if not complete_if_empty:
raise Exception("No iterable value found for the loop block")
+ return
# register the loop in the global context
block_validation_output.context.parent_workflow_run_block_id = workflow_run_block_id
diff --git a/skyvern/webeye/schemas.py b/skyvern/webeye/schemas.py
index 2b5ba7cad..89ab9cd8f 100644
--- a/skyvern/webeye/schemas.py
+++ b/skyvern/webeye/schemas.py
@@ -112,6 +112,9 @@ class BrowserSessionResponse(BaseModel):
# endpoints serialize an unpaginated set concurrently. Only the single-session fetch —
# the one live view actually reads — pays for it.
include_stream_transport: bool = False,
+ # The fan-out endpoints already gather across sessions, so overlapping one session's two
+ # listings there only doubles the request's peak pool checkouts; the polled GET opts in.
+ concurrent_listings: bool = False,
) -> BrowserSessionResponse:
"""
Creates a BrowserSessionResponse from a PersistentBrowserSession object.
@@ -132,40 +135,60 @@ class BrowserSessionResponse(BaseModel):
downloaded_files: list[FileInfo] = []
recordings: list[FileInfo] = []
if storage:
- try:
- async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
- downloaded_files = await storage.get_shared_downloaded_files_in_browser_session(
- organization_id=browser_session.organization_id,
- browser_session_id=browser_session.persistent_browser_session_id,
- )
- except asyncio.TimeoutError:
- LOG.warning(
- "Timeout getting downloaded files", browser_session_id=browser_session.persistent_browser_session_id
- )
- if fail_download_lookup:
- raise HTTPException(
- status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
- detail={"code": "downloaded_files_unavailable", "retryable": True},
- ) from None
-
- try:
- async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
- recordings = await storage.get_shared_recordings_in_browser_session(
- organization_id=browser_session.organization_id,
+
+ async def list_downloads() -> list[FileInfo]:
+ try:
+ async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
+ return await storage.get_shared_downloaded_files_in_browser_session(
+ organization_id=browser_session.organization_id,
+ browser_session_id=browser_session.persistent_browser_session_id,
+ )
+ except asyncio.TimeoutError:
+ LOG.warning(
+ "Timeout getting downloaded files",
browser_session_id=browser_session.persistent_browser_session_id,
)
- if recordings:
- recordings = await app.AGENT_FUNCTION.select_browser_session_recordings(
+ if fail_download_lookup:
+ raise HTTPException(
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+ detail={"code": "downloaded_files_unavailable", "retryable": True},
+ ) from None
+ return []
+
+ async def list_recordings() -> list[FileInfo]:
+ try:
+ async with asyncio.timeout(GET_DOWNLOADED_FILES_TIMEOUT):
+ listed = await storage.get_shared_recordings_in_browser_session(
organization_id=browser_session.organization_id,
browser_session_id=browser_session.persistent_browser_session_id,
- recordings=recordings,
- browser_vendor=browser_session.browser_vendor,
)
- except asyncio.TimeoutError:
- LOG.warning(
- "Timeout getting recordings", browser_session_id=browser_session.persistent_browser_session_id
- )
- recordings = []
+ if listed:
+ listed = await app.AGENT_FUNCTION.select_browser_session_recordings(
+ organization_id=browser_session.organization_id,
+ browser_session_id=browser_session.persistent_browser_session_id,
+ recordings=listed,
+ browser_vendor=browser_session.browser_vendor,
+ )
+ return listed
+ except asyncio.TimeoutError:
+ LOG.warning(
+ "Timeout getting recordings", browser_session_id=browser_session.persistent_browser_session_id
+ )
+ return []
+
+ if concurrent_listings:
+ # Downloads is awaited first so its outcome still decides the response before recordings can.
+ recordings_task = asyncio.create_task(list_recordings())
+ try:
+ downloaded_files = await list_downloads()
+ recordings = await recordings_task
+ finally:
+ # A downloads failure or a cancelled request must not leave the recordings listing running.
+ recordings_task.cancel()
+ await asyncio.gather(recordings_task, return_exceptions=True)
+ else:
+ downloaded_files = await list_downloads()
+ recordings = await list_recordings()
# Sort downloaded files by modified_at in descending order (newest first)
# Treat None as "oldest".
diff --git a/tests/unit/fake_workflow_run_context.py b/tests/unit/fake_workflow_run_context.py
index 40773ed6e..7df758615 100644
--- a/tests/unit/fake_workflow_run_context.py
+++ b/tests/unit/fake_workflow_run_context.py
@@ -9,7 +9,7 @@ from __future__ import annotations
from typing import Any
-from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+from skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext
class FakeWorkflowRunContext:
@@ -36,6 +36,9 @@ class FakeWorkflowRunContext:
self.organization_id: str | None = None
self.browser_session_id: str | None = None
self.workflow_run_outputs: dict[str, Any] = dict(workflow_run_outputs or {})
+ self.block_outcomes: dict[str, BlockOutcome] = {}
+ self.carried_block_labels: set[str] = set()
+ self.parameters: dict[str, Any] = {}
self.workflow = None
def get_block_metadata(self, label: str | None) -> dict[str, Any]:
@@ -75,5 +78,7 @@ class FakeWorkflowRunContext:
return self.secrets[secret_id_or_value]
return None
- # Borrow the real implementation so scoping behavior cannot drift from production.
+ # Borrow the real implementations so scoping and outcome behavior cannot drift from production.
credential_template_entries = WorkflowRunContext.credential_template_entries
+ record_block_outcome = WorkflowRunContext.record_block_outcome
+ get_block_outcome = WorkflowRunContext.get_block_outcome
diff --git a/tests/unit/force_stub_app.py b/tests/unit/force_stub_app.py
index 6200bac82..db11a77ab 100644
--- a/tests/unit/force_stub_app.py
+++ b/tests/unit/force_stub_app.py
@@ -139,6 +139,9 @@ def create_forge_stub_app() -> ForgeApp:
fake_app_module.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
fake_app_module.STORAGE = _LazyNamespace()
fake_app_module.CACHE = _LazyNamespace()
+ # The real app starts with no bearer authentication; without the attributes a bearer raises AttributeError.
+ fake_app_module.authentication_function = None
+ fake_app_module.authenticate_user_function = None
return fake_app_module
diff --git a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py
index a52eaa32d..42a5883bd 100644
--- a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py
+++ b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py
@@ -163,6 +163,7 @@ def test_a_turn_without_browser_authority_advertises_no_browser_tool() -> None:
"get_org_workflow",
"list_workflow_schedules",
"get_workflow_schedule",
+ "list_workflow_runs",
)
assert withheld.alias_map == {name: aliases[name] for name in withheld.ordered_mcp_names}
assert withheld.overlays == {name: overlays[name] for name in withheld.ordered_mcp_names}
diff --git a/tests/unit/forge/sdk/routes/test_browser_sessions.py b/tests/unit/forge/sdk/routes/test_browser_sessions.py
index 44a01240f..f6c77965d 100644
--- a/tests/unit/forge/sdk/routes/test_browser_sessions.py
+++ b/tests/unit/forge/sdk/routes/test_browser_sessions.py
@@ -255,9 +255,37 @@ async def test_get_browser_session_enables_strict_download_lookup() -> None:
app_mock.STORAGE,
fail_download_lookup=True,
include_stream_transport=True,
+ concurrent_listings=True,
)
[email protected]("endpoint", ["history", "active"])
+async def test_fan_out_listings_keep_each_sessions_storage_listings_sequential(endpoint: str) -> None:
+ """These endpoints already gather across sessions, so overlapping a session's two listings would
+ only raise the request's peak pool checkouts from N to 2N."""
+ sessions = [MagicMock(), MagicMock()]
+ app_mock = MagicMock()
+ app_mock.DATABASE.browser_sessions.get_persistent_browser_sessions_history = AsyncMock(return_value=sessions)
+ app_mock.PERSISTENT_SESSIONS_MANAGER.get_active_sessions = AsyncMock(return_value=sessions)
+ from_browser_session = AsyncMock(return_value=MagicMock())
+
+ with (
+ patch.object(browser_sessions_mod, "app", app_mock),
+ patch.object(browser_sessions_mod.BrowserSessionResponse, "from_browser_session", from_browser_session),
+ ):
+ if endpoint == "history":
+ await browser_sessions_mod.get_browser_sessions_all(
+ current_org=SimpleNamespace(organization_id="org_1"), page=1, page_size=100
+ )
+ else:
+ await browser_sessions_mod.get_browser_sessions(current_org=SimpleNamespace(organization_id="org_1"))
+
+ assert from_browser_session.await_count == len(sessions)
+ for call in from_browser_session.await_args_list:
+ assert call.kwargs.get("concurrent_listings", False) is False
+
+
@pytest.mark.asyncio
async def test_create_browser_session_records_the_caller_as_creator(
monkeypatch: pytest.MonkeyPatch, sqlite_engine: AsyncEngine
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 043daa0d1..11d6c828f 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -67,11 +67,12 @@ from skyvern.forge.taskv3 import engine as taskv3_engine
from skyvern.forge.taskv3 import tools as taskv3_tools
from skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationState
from skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loop
-from skyvern.forge.taskv3.goal_composition import PAGE_DATA_NOTE
+from skyvern.forge.taskv3.goal_composition import PAGE_DATA_NOTE, CodeProgressRecord
from skyvern.forge.taskv3.handoff_redaction import pin_caller_authored_block_urls
from skyvern.forge.taskv3.loop import (
ACTION_LOOP_GUARD,
NAV_DEAD_END_GUARD,
+ TOKEN_BUDGET_EXTENDED_EVENT,
LoopOutcome,
RoundAction,
ToolSpec,
@@ -103,6 +104,7 @@ from skyvern.webeye.actions.actions import (
from tests.unit.helpers import make_action_row, make_browser_state, make_organization, make_step, make_task
from tests.unit.scoped_asyncio import ScopedAsyncio
from tests.unit.test_taskv3_engine import _fixed_read_tool, _ReaskAnsweringCaller
+from tests.unit.test_taskv3_loop import _ScriptedCaller
from tests.unit.test_taskv3_tools import _FakePage, _fixed_page_provider
@@ -131,6 +133,7 @@ async def _run_execute_task_v3(
own_block_row: WorkflowRunBlock | None = None,
own_block_lookup_raises: BaseException | None = None,
workflow_owned_recovery: bool = False,
+ recovery_code_progress: CodeProgressRecord | None = None,
landed_url: str | None = None,
navigation_status: int | None = None,
page_url_after_settle: str | None = None,
@@ -142,6 +145,8 @@ async def _run_execute_task_v3(
# Called with the loop's kwargs before the loop returns, to drive state the loop's tools own.
on_loop: Callable[[dict[str, Any]], None] | None = None,
on_loop_async: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
+ # Replaces the canned `outcome`: receives the kwargs agent.py built and returns the loop's outcome.
+ loop_body: Callable[[dict[str, Any]], Awaitable[LoopOutcome]] | None = None,
# A prompt the fake loop hands the goal judge, the way the finish gate would, when one was built.
goal_judge_prompt: str | None = None,
# Leave the credential-TOTP candidate gate reading the real workflow-run context.
@@ -195,6 +200,8 @@ async def _run_execute_task_v3(
on_loop(kwargs)
if on_loop_async is not None:
await on_loop_async(kwargs)
+ if loop_body is not None:
+ return await loop_body(kwargs)
if goal_judge_prompt is not None and kwargs.get("goal_judge") is not None:
page.is_closed = MagicMock(return_value=False)
loop_mock.goal_judge_response = await kwargs["goal_judge"](goal_judge_prompt)
@@ -289,6 +296,7 @@ async def _run_execute_task_v3(
task_block=task_block,
workflow_owned_recovery=workflow_owned_recovery,
recovery_credential_parameter_keys=recovery_credential_parameter_keys,
+ recovery_code_progress=recovery_code_progress,
)
finally:
skyvern_context.reset()
@@ -2013,6 +2021,7 @@ async def _run_execute_step_gate(
experimentation_provider: BaseExperimentationProvider | None = None,
workflow_run: Any = None,
workflow_owned_recovery: bool = False,
+ recovery_code_progress: CodeProgressRecord | None = None,
**task_overrides: Any,
) -> tuple[AsyncMock, AsyncMock]:
"""Drive ForgeAgent.execute_step through the v3 dispatch gate and return (mocked _execute_task_v3,
@@ -2068,6 +2077,7 @@ async def _run_execute_step_gate(
engine=engine,
task_block=task_block,
workflow_owned_recovery=workflow_owned_recovery,
+ recovery_code_progress=recovery_code_progress,
download_baseline_files=[],
)
except _StepEngineDispatched:
@@ -2212,6 +2222,22 @@ async def test_an_unevaluable_kill_switch_runs_an_explicit_v3_block_on_the_step_
)
+async def test_execute_step_hands_the_recovery_code_outline_to_the_v3_run() -> None:
+ record = CodeProgressRecord(before=(), failed_step="click export", failed_line=3, after=())
+
+ v3_mock, _step_engine_mock = await _run_execute_step_gate(
+ engine=agent_module.RunEngine.skyvern_v3,
+ task_block=None,
+ workflow_owned_recovery=True,
+ recovery_code_progress=record,
+ workflow_run_id="wr_recovery_outline",
+ )
+
+ v3_mock.assert_awaited_once()
+ assert v3_mock.await_args.kwargs["recovery_code_progress"] is record
+
+
@pytest.mark.asyncio
async def test_a_kill_switched_recovery_fails_instead_of_running_on_the_step_engine() -> None:
provider = MagicMock(spec=BaseExperimentationProvider)
@@ -2970,6 +2996,70 @@ async def test_execute_task_v3_atomic_block_ceiling_pinned_to_its_own_cap(monkey
assert loop_mock.await_args.kwargs["max_action_steps_ceiling"] == 5
+class _AdvancingFormPage(_FakePage):
+ """Every action advances the form, so the next observe is fresh page-change evidence."""
+
+ async def evaluate(self, _js: str) -> str:
+ raw = await super().evaluate(_js)
+ if "document.readyState" in _js:
+ return raw
+ section = sum(1 for name, _args in self.calls if name in ("click", "fill", "type"))
+ data = json.loads(raw)
+ data["title"] = f"Apply, section {section}"
+ data["elements"][0]["label"] = f"Section {section} first name"
+ return json.dumps(data)
+
+
+async def test_execute_task_v3_navigation_block_token_trip_with_progress_is_extended(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A long form resends a growing transcript, so the token backstop trips while most of the step budget is
+ # unspent. Driven through agent.py with the real engine and loop, so the step floor, ceilings and backstops
+ # agent.py configures are the ones the grant must survive. The form is filled one field per round, and each
+ # new field followed by a changed page renews the grant.
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for i in range(5):
+ script.append([("observe", {})])
+ script.append([("type", {"selector": f"#field-{i}", "text": f"answer {i}"})])
+ script.append([("observe", {})])
+ script.append([("click", {"selector": "#submit"})])
+ script.append([("finish", {"status": "completed", "reason": "submitted"})])
+ caller = _ScriptedCaller(script, turn_tokens=200_000)
+ page = _AdvancingFormPage()
+ loop_kwargs: dict[str, Any] = {}
+
+ async def _real_loop(kwargs: dict[str, Any]) -> LoopOutcome:
+ loop_kwargs.update(kwargs)
+ return await run_task_v3_agent_loop(
+ # The scripted caller takes no step-scoped call kwargs.
+ **{**kwargs, "page_provider": _fixed_page_provider(page), "llm_caller": caller, "step": None}
+ )
+
+ monkeypatch.setattr(ForgeAgent, "_check_workflow_run_step_budget", AsyncMock(return_value=None))
+ with capture_logs() as logs:
+ _step, task, _loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch,
+ LoopOutcome(status="failed", reason="the canned outcome must not be used"),
+ task_block=_make_block(NavigationBlock, navigation_goal="Fill and submit the application"),
+ workflow_run_id="wr_long_form",
+ max_steps_per_run=10,
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ loop_body=_real_loop,
+ working_page=page,
+ )
+ assert loop_kwargs["max_action_steps"] == MIN_ACTION_STEPS
+ granted = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT]
+ assert [(entry["guard"], entry["original_max_tokens"]) for entry in granted] == [
+ ("max_tokens", loop_kwargs["max_tokens"]),
+ ("max_tokens", granted[0]["max_tokens"]),
+ ]
+ assert granted[0]["action_steps"] < MIN_ACTION_STEPS
+ assert caller.calls == len(script)
+ assert task.status == TaskStatus.completed
+
+
@pytest.mark.asyncio
async def test_execute_task_v3_no_workflow_ceiling_without_a_pool(monkeypatch: pytest.MonkeyPatch) -> None:
# No org pool -> no hard ceiling: the loop's extension is bounded only by its own gate.
@@ -3145,6 +3235,35 @@ async def test_execute_task_v3_should_cancel_skips_workflow_read_for_bare_task(
# ---------------------------------------------------------------------------
[email protected]("customer_precedence", [False, True], ids=["one_compose_pass", "recompose_pass"])
+async def test_a_recovery_code_outline_reaches_the_model_goal_but_never_the_task_row(
+ monkeypatch: pytest.MonkeyPatch, customer_precedence: bool
+) -> None:
+ monkeypatch.setattr(settings, "TASK_V3_CUSTOMER_PRECEDENCE", customer_precedence)
+ record = CodeProgressRecord(
+ before=("open the portal",), failed_step="click the invoices tab", failed_line=2, after=("download",)
+ )
+
+ _step, task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch,
+ LoopOutcome(status="completed", reason="done", billable_actions=[]),
+ workflow_owned_recovery=True,
+ recovery_code_progress=record,
+ workflow_run_id="wr_recovery_outline",
+ navigation_goal="Download the latest invoice",
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+
+ goal = loop_mock.await_args.kwargs["goal"]
+ assert goal.startswith("(This goal contains a value of unverified origin") is customer_precedence
+ assert goal.count("Code outline") == 1
+ assert goal.endswith("- Later in the code: download")
+ assert task.navigation_goal == "Download the latest invoice"
+ assert "Code outline" not in str(loop_mock.update_task_kwargs)
+
+
@pytest.mark.asyncio
async def test_a_workflow_owned_recovery_keeps_its_caller_s_step_cap(monkeypatch: pytest.MonkeyPatch) -> None:
# The code block AI fallback sizes its own budget from the org's cap, so the v3 floor must not
diff --git a/tests/unit/test_context_manager.py b/tests/unit/test_context_manager.py
index 4a969b085..27eb07da0 100644
--- a/tests/unit/test_context_manager.py
+++ b/tests/unit/test_context_manager.py
@@ -35,11 +35,12 @@ from skyvern.forge.sdk.services.credential.custom_credential_vault_service impor
CustomCredentialNotConfiguredError,
)
from skyvern.forge.sdk.workflow import context_manager as cm
-from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+from skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext
from skyvern.forge.sdk.workflow.credential_fetch_outcome import (
RUN_CREDENTIAL_FETCH_FINISHED_MESSAGE,
classify_credential_fetch_failure,
)
+from skyvern.forge.sdk.workflow.models.block import BranchEvaluationContext, WaitBlock
from skyvern.forge.sdk.workflow.models.parameter import (
AzureVaultCredentialParameter,
BitwardenLoginCredentialParameter,
@@ -47,6 +48,8 @@ from skyvern.forge.sdk.workflow.models.parameter import (
WorkflowParameterType,
)
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition, WorkflowRunParameter
+from skyvern.schemas.workflows import BlockStatus
+from tests.unit.conftest import make_block_output_parameter
from tests.unit.fake_workflow_run_context import FakeWorkflowRunContext
from tests.unit.scoped_asyncio import ScopedAsyncio
@@ -524,3 +527,56 @@ class TestRunCredentialFetchOutcome:
self, error: BaseException, customer_owned: bool, outcome: str
) -> None:
assert classify_credential_fetch_failure(error, customer_owned=customer_owned)[0] == outcome
+
+
+def _outcome_context() -> WorkflowRunContext:
+ return WorkflowRunContext(
+ workflow_title="Outcome test",
+ workflow_id="workflow-id",
+ workflow_permanent_id="wpid",
+ workflow_run_id="run-id",
+ aws_client=AsyncMock(),
+ )
+
+
+def test_block_outcome_masks_secrets_before_bounding_the_reason() -> None:
+ context = _outcome_context()
+ context.secrets["placeholder_pw"] = "hunter2secret"
+ # The secret straddles the bound: cutting first would leave its head in the stored reason.
+ reason = "a" * (cm.BLOCK_OUTCOME_FAILURE_REASON_MAX_CHARS - 5) + "hunter2secret" + "b" * 100
+
+ context.record_block_outcome("login", BlockStatus.failed, ["AUTH_FAILURE"], reason)
+
+ outcome = context.get_block_outcome("login")
+ assert outcome is not None
+ assert outcome.status is BlockStatus.failed
+ assert outcome.error_codes == ["AUTH_FAILURE"]
+ assert outcome.failure_reason is not None
+ assert len(outcome.failure_reason) == cm.BLOCK_OUTCOME_FAILURE_REASON_MAX_CHARS
+ assert "hunte" not in outcome.failure_reason
+ assert context.get_block_outcome("never_ran") is None
+
+
+async def test_block_outcome_is_invisible_to_templates_and_the_branch_snapshot() -> None:
+ context = _outcome_context()
+ login_output = make_block_output_parameter("login_output")
+ await context.register_output_parameter_value_post_execution(
+ login_output, {"status": "completed", "extracted_information": {"user": "ada"}}
+ )
+ block = WaitBlock(label="login", output_parameter=login_output, wait_sec=1)
+ template = "{{ login }} | {{ login_output }} | {{ workflow_run_outputs }}"
+ branch_context = BranchEvaluationContext(workflow_run_context=context, block_label="login")
+
+ def observe() -> tuple[str, str]:
+ rendered = block.format_block_parameter_template_from_workflow_run_context(template, context)
+ snapshot = json.dumps(branch_context.build_llm_safe_context_snapshot(), sort_keys=True, default=str)
+ return rendered, snapshot
+
+ before = observe()
+ context.record_block_outcome("login", BlockStatus.failed, ["AUTH_FAILURE"], "wrong password")
+
+ assert observe() == before
+ assert context.get_block_outcome("login") == BlockOutcome(
+ status=BlockStatus.failed, error_codes=["AUTH_FAILURE"], failure_reason="wrong password"
+ )
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index 9886c71f6..5381f7a73 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -915,10 +915,10 @@ class TestMCPToolOverlayCompleteness:
"disable_workflow_schedule",
"cancel_workflow_schedule",
"delete_workflow_schedule",
+ "list_workflow_runs",
}
assert set(alias_map.keys()) == expected_aliases
assert all(v.startswith("skyvern_") for v in alias_map.values())
- assert "skyvern_workflow_run_list" not in alias_map.values()
assert "query" in tools_module._build_skyvern_mcp_overlays()["list_org_workflows"].hide_params
def test_every_alias_has_overlay(self) -> None:
diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.py
index d78094ea4..066115ede 100644
--- a/tests/unit/test_copilot_mcp_adapter.py
+++ b/tests/unit/test_copilot_mcp_adapter.py
@@ -3075,6 +3075,90 @@ async def test_schedule_tools_always_target_the_chat_workflow(tool_name: str, mo
]
[email protected]("browser_tools_available", [True, False])
+async def test_run_list_is_offered_without_a_workflow_id_and_with_a_status_filter(
+ browser_tools_available: bool,
+) -> None:
+ tools, _ = await _listed_tools(browser_tools_available=browser_tools_available)
+
+ properties = tools["list_workflow_runs"].inputSchema["properties"]
+ assert "workflow_id" not in properties
+ assert "search_key" not in properties
+ assert "error_code" not in properties
+ assert "status" in properties
+ assert "workflow_id" not in tools["list_workflow_runs"].inputSchema.get("required", [])
+
+
+async def test_run_list_always_targets_the_chat_workflow(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat")
+ server = _schedule_server(monkeypatch, ctx, {"ok": True, "data": {"runs": []}})
+
+ await server.call_tool("list_workflow_runs", {"workflow_id": "wpid_foreign", "status": ["terminated"]})
+
+ assert server._client.calls == [
+ ("skyvern_workflow_run_list", {"workflow_id": "wpid_chat", "status": ["terminated"]}),
+ ]
+
+
+async def test_run_list_shows_the_model_only_the_fields_that_pick_a_run(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat")
+ run = {
+ "run_id": "wr_1",
+ "status": "terminated",
+ "created_at": "2026-09-24T19:30:00Z",
+ "trigger_type": "scheduled",
+ "copilot_session_id": None,
+ "run_type": "workflow_run",
+ "failure_reason": "Login rejected for password hunter2-unseen",
+ "output_summary": {"present": True, "scalar_preview": {"api_token": "hunter2-unseen"}},
+ "artifact_summary": {"downloaded_file_names": ["hunter2-unseen.pdf"]},
+ "run_with": "agent",
+ }
+ server = _schedule_server(monkeypatch, ctx, {"ok": True, "data": {"runs": [run], "count": 1, "has_more": False}})
+
+ result = await server.call_tool("list_workflow_runs", {"status": ["terminated"]})
+
+ surfaced = json.loads(result.content[0].text)
+ assert surfaced["data"]["runs"] == [
+ {"run_id": "wr_1", "status": "terminated", "created_at": "2026-09-24T19:30:00Z", "trigger_type": "scheduled"}
+ ]
+ assert "hunter2-unseen" not in result.content[0].text
+
+
+ ("tool_name", "args", "data"),
+ [
+ (
+ "list_workflow_runs",
+ {"status": ["terminated"]},
+ {"runs": [{"run_id": "wr_1", "failure_reason": "password hunter2-unseen"}]},
+ ),
+ (
+ "get_workflow_schedule",
+ {"workflow_schedule_id": "wfs_1"},
+ {"workflow_schedule_id": "wfs_1", "parameters": {"password": "hunter2-unseen"}},
+ ),
+ ],
+)
+async def test_a_crashed_stored_value_filter_withholds_the_result(
+ tool_name: str, args: dict[str, Any], data: dict[str, Any], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat")
+ server = _schedule_server(monkeypatch, ctx, {"ok": True, "data": data})
+ server._overlays[tool_name] = replace(server._overlays[tool_name], post_hook=_crash)
+
+ result = await server.call_tool(tool_name, args)
+
+ assert result.isError is True
+ text = " ".join(content.text for content in result.content)
+ assert "withheld" in text
+ assert "hunter2-unseen" not in text
+
+
@pytest.mark.asyncio
async def test_schedule_create_is_refused_while_the_turn_holds_an_unsaved_proposal(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.py
index 38f7febc2..fa78ad76f 100644
--- a/tests/unit/test_copilot_model_telemetry.py
+++ b/tests/unit/test_copilot_model_telemetry.py
@@ -370,6 +370,46 @@ async def test_explicit_cache_envelope_leaves_other_routes_unchanged(
assert "extra_body" not in requests[0]
[email protected]("model_name", ["anthropic/claude-sonnet-5-5", "bedrock/global.anthropic.claude-sonnet-5-5"])
[email protected]("stream", [False, True])
+async def test_cacheable_system_instructions_survive_provider_message_copy(
+ monkeypatch: pytest.MonkeyPatch,
+ model_name: str,
+ stream: bool,
+) -> None:
+ prompt = CacheableSystemInstructions("stable instructions", "\ndynamic context", cache_namespace="wcc_test")
+ copied_messages: list[list[dict[str, Any]]] = []
+
+ async def fake_acompletion(**kwargs: Any) -> LiteLLMModelResponse | AsyncStream[ChatCompletionChunk]:
+ copied_messages.append(copy.deepcopy(kwargs["messages"]))
+ if kwargs.get("stream"):
+ return cast(AsyncStream[ChatCompletionChunk], _ChunkStream(_stream_chunks()))
+ return _completion()
+
+ monkeypatch.setattr("litellm.acompletion", fake_acompletion)
+ model = CopilotLitellmModel(model=model_name, next_model_call_index=lambda: 1)
+ if stream:
+ events = [
+ event
+ async for event in model.stream_response(
+ system_instructions=prompt,
+ input=[{"role": "user", "content": "Return 42"}],
+ model_settings=ModelSettings(include_usage=True),
+ tools=[],
+ output_schema=None,
+ handoffs=[],
+ tracing=ModelTracing.DISABLED,
+ )
+ ]
+ response = events[-1].response
+ else:
+ response = await _get_response(model, system_instructions=prompt)
+
+ assert copied_messages[0][0] == {"role": "system", "content": str(prompt)}
+ assert ItemHelpers.extract_last_text(response.output[-1]) == "42"
+
+
@pytest.mark.asyncio
async def test_stream_capture_preserves_events_order_and_backpressure(monkeypatch: pytest.MonkeyPatch) -> None:
requests: list[bytes] = []
diff --git a/tests/unit/test_handle_script_termination.py b/tests/unit/test_handle_script_termination.py
index 8b80a5b1d..75e0b9864 100644
--- a/tests/unit/test_handle_script_termination.py
+++ b/tests/unit/test_handle_script_termination.py
@@ -22,7 +22,7 @@ from skyvern.forge.sdk.db.repositories.tasks import TasksRepository
from skyvern.forge.sdk.db.repositories.workflows import WorkflowsRepository
from skyvern.forge.sdk.models import StepStatus
from skyvern.forge.sdk.schemas.tasks import TaskStatus
-from skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager, WorkflowRunContext
+from skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowContextManager, WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import NavigationBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition
@@ -206,6 +206,7 @@ async def test_cached_creation_and_termination_persist_contract(
assert (created.organization_id, created.workflow_run_id, created.attempt_number) == ("o_cached", "wr_cached", 3)
existing_error = {"error_code": "existing", "reasoning": "Earlier condition", "confidence_float": 0.5}
await tasks.update_task(task_id, organization_id="o_cached", errors=[existing_error])
+ await observer.update_workflow_run_block(block_id, organization_id="o_cached", error_codes=["existing"])
error = UserDefinedError(error_code="unavailable", reasoning="Requested item unavailable", confidence_float=0.9)
run_context = RunContext(parameters={}, page=Mock())
run_context.actions_and_results.append((TerminateAction(errors=[error]), []))
@@ -230,3 +231,8 @@ async def test_cached_creation_and_termination_persist_contract(
block = await observer.get_workflow_run_block(block_id, organization_id="o_cached")
assert block.status == BlockStatus.terminated
assert block.attempt_number == 3
+ assert block.error_codes == ["unavailable"]
+ # The cached path writes the same outcome record the engine writes for an agent-run block.
+ assert workflow_context.get_block_outcome("target") == BlockOutcome(
+ status=BlockStatus.terminated, error_codes=["unavailable"], failure_reason="Requested item unavailable"
+ )
diff --git a/tests/unit/test_mcp_workflow_route_prefixes.py b/tests/unit/test_mcp_workflow_route_prefixes.py
index cf79880f4..03fb1e8f9 100644
--- a/tests/unit/test_mcp_workflow_route_prefixes.py
+++ b/tests/unit/test_mcp_workflow_route_prefixes.py
@@ -17,7 +17,6 @@ from __future__ import annotations
import ast
import inspect
-from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -105,34 +104,6 @@ async def test_retry_workflow_run_bad_request_uses_detail(monkeypatch: pytest.Mo
assert result["error"]["message"] == "Workflow run wr_x is not terminal"
-async def test_workflow_run_list_fetches_extra_row_for_has_more(monkeypatch: pytest.MonkeyPatch) -> None:
- list_mock = AsyncMock(
- return_value=[
- SimpleNamespace(workflow_run_id="wr_1", status="completed"),
- SimpleNamespace(workflow_run_id="wr_2", status="completed"),
- SimpleNamespace(workflow_run_id="wr_3", status="completed"),
- ]
- )
- monkeypatch.setattr(workflow_tools, "list_workflow_runs_raw", list_mock)
-
- result = await workflow_tools.skyvern_workflow_run_list("wpid_x", page=1, page_size=2)
-
- assert result["ok"] is True
- list_mock.assert_awaited_once_with(
- "wpid_x",
- page=1,
- page_size=3,
- status=None,
- search_key=None,
- error_code=None,
- include_child_runs=False,
- )
- assert result["data"]["count"] == 2
- assert result["data"]["has_more"] is True
- assert [run["run_id"] for run in result["data"]["runs"]] == ["wr_1", "wr_2"]
-
-
@pytest.mark.asyncio
@pytest.mark.parametrize(("include_child_runs", "expected_param"), [(False, None), (True, True)])
async def test_workflow_run_list_sends_include_child_runs_only_when_opted_in(
diff --git a/tests/unit/test_mcp_workflow_tools.py b/tests/unit/test_mcp_workflow_tools.py
index 76f7e0785..87f700265 100644
--- a/tests/unit/test_mcp_workflow_tools.py
+++ b/tests/unit/test_mcp_workflow_tools.py
@@ -13,6 +13,7 @@ import skyvern.cli.mcp_tools.workflow as workflow_tools
from skyvern.cli.core.result import set_concise_responses
from skyvern.cli.mcp_tools import mcp
from skyvern.cli.mcp_tools.response import MCP_MAX_RESPONSE_CHARS, size_capped
+from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
from skyvern.schemas.workflows import (
WorkflowCreateYAMLRequest,
WorkflowRequest,
@@ -4772,3 +4773,61 @@ async def test_registered_large_workflow_get_reports_skills_with_overflow_hint(
assert f"carries {len(prompt)} persisted skill-instruction characters" in skills["hint"]
assert "--id wpid_test --version 7 --definition-file wf.json" in skills["hint"]
assert "data.workflow_definition.workflow_system_prompt" not in skills["hint"]
+
+
+ "run",
+ [
+ {
+ "workflow_run_id": "wr_api",
+ "status": "terminated",
+ "created_at": "2026-09-20T08:30:00+00:00",
+ "trigger_type": "api",
+ "copilot_session_id": "wcc_1",
+ },
+ SimpleNamespace(
+ workflow_run_id="wr_api",
+ status="terminated",
+ created_at=datetime(2026, 9, 20, 8, 30, tzinfo=timezone.utc),
+ trigger_type=WorkflowRunTriggerType.api,
+ copilot_session_id="wcc_1",
+ ),
+ ],
+ ids=["api_json", "typed"],
+)
+async def test_workflow_run_list_summary_carries_when_and_how_each_run_started(
+ run: object, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(workflow_tools, "list_workflow_runs_raw", AsyncMock(return_value=[run]))
+
+ result = await workflow_tools.skyvern_workflow_run_list("wpid_x", status=["terminated"])
+
+ summary = json.loads(json.dumps(result))["data"]["runs"][0]
+ assert summary["created_at"] == "2026-09-20T08:30:00+00:00"
+ assert summary["trigger_type"] == "api"
+ assert summary["copilot_session_id"] == "wcc_1"
+
+
[email protected]("run_count", [13, 10])
+async def test_workflow_run_list_pages_through_every_run_in_order(
+ run_count: int, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ run_ids = [f"wr_{i}" for i in range(run_count)]
+
+ async def list_runs(workflow_id: str, *, page: int, page_size: int, **_: object) -> list[dict[str, str]]:
+ offset = (page - 1) * page_size
+ return [{"workflow_run_id": run_id, "status": "terminated"} for run_id in run_ids[offset : offset + page_size]]
+
+ monkeypatch.setattr(workflow_tools, "list_workflow_runs_raw", list_runs)
+
+ seen: list[str] = []
+ for page in range(1, 10):
+ data = (await workflow_tools.skyvern_workflow_run_list("wpid_x", page=page, page_size=5))["data"]
+ assert data["runs"], f"has_more promised page {page}, which was empty"
+ seen.extend(run["run_id"] for run in data["runs"])
+ if not data["has_more"]:
+ break
+
+ assert seen == run_ids
diff --git a/tests/unit/test_org_auth_service.py b/tests/unit/test_org_auth_service.py
index 669cf99dd..c03484be5 100644
--- a/tests/unit/test_org_auth_service.py
+++ b/tests/unit/test_org_auth_service.py
@@ -1,26 +1,33 @@
import asyncio
from collections.abc import Iterator
+from dataclasses import asdict
from datetime import UTC, datetime, timedelta
from time import time as current_time
from types import SimpleNamespace
+from unittest.mock import MagicMock
import jwt
import pytest
-from fastapi import Depends, FastAPI, HTTPException
+from fastapi import Depends, FastAPI, HTTPException, WebSocket
from fastapi.testclient import TestClient
from freezegun import freeze_time
from skyvern.config import settings
+from skyvern.forge import request_logging
from skyvern.forge.agent_functions import AgentFunction
+from skyvern.forge.request_logging import RequestLoggingMiddleware
from skyvern.forge.sdk.core.organization_age_cache import cached_org_age
from skyvern.forge.sdk.core.security import create_access_token
from skyvern.forge.sdk.routes.routers import legacy_base_router
+from skyvern.forge.sdk.routes.streaming.auth import auth as streaming_auth
from skyvern.forge.sdk.schemas.organizations import Organization, OrganizationAuthToken, OrganizationAuthTokenType
-from skyvern.forge.sdk.services import org_auth_service, org_auth_token_service
+from skyvern.forge.sdk.services import org_auth_service, org_auth_token_service, request_principal
from skyvern.forge.sdk.services.org_auth_service import (
+ CallerContext,
_get_api_key_debug_fields,
_normalize_api_key_with_flags,
)
+from skyvern.forge.sdk.services.request_principal import BearerIdentity, get_request_principal
def test_authenticated_org_age_is_cached_for_lines_that_only_name_the_org() -> None:
@@ -1093,3 +1100,327 @@ def test_bearer_scheme_is_matched_case_insensitively() -> None:
assert org_auth_service._extract_bearer_token("Basic token-canary") is None
assert org_auth_service._extract_bearer_token("token-canary") is None
assert org_auth_service._extract_bearer_token("Bearer ") is None
+
+
+_PRINCIPAL_FIELDS = ("auth_kind", "user_id", "org_role", "org_role_claim")
+_NO_USER = {"user_id": None, "org_role": None, "org_role_claim": None}
+_ADMIN = {"user_id": "user_admin", "org_role": "org:admin", "org_role_claim": "org_role"}
+_MEMBER = {"user_id": "user_member", "org_role": "member", "org_role_claim": "o.rol"}
+_ROLELESS = {"user_id": "user_roleless", "org_role": None, "org_role_claim": None}
+
+
+class _StubIdentityProvider(AgentFunction):
+ """Stands in for a deployment's identity provider: maps a bearer token to the user it proves."""
+
+ def __init__(self) -> None:
+ self.resolved: list[tuple[str, str]] = []
+
+ async def resolve_bearer_identity(self, bearer_token: str, organization_id: str) -> BearerIdentity | None:
+ self.resolved.append((bearer_token, organization_id))
+ if bearer_token == "stale-bearer":
+ raise HTTPException(status_code=403, detail="Auth token is expired")
+ if bearer_token == "broken-bearer":
+ raise RuntimeError("identity provider lookup failed")
+ identity = {"admin-bearer": _ADMIN, "member-bearer": _MEMBER, "roleless-bearer": _ROLELESS}.get(bearer_token)
+ return BearerIdentity(**identity) if identity else None
+
+
+def _principal_app() -> FastAPI:
+ fastapi_app = FastAPI()
+ fastapi_app.add_middleware(RequestLoggingMiddleware)
+
+ def principal() -> dict[str, str | None] | None:
+ resolved = get_request_principal()
+ return asdict(resolved) if resolved else None
+
+ @fastapi_app.post("/org")
+ async def org(_: Organization = Depends(org_auth_service.get_current_org)) -> dict[str, str | None] | None:
+ return principal()
+
+ @fastapi_app.post("/credential")
+ async def credential(
+ _: Organization = Depends(org_auth_service.get_current_org_for_credential_routes),
+ ) -> dict[str, str | None] | None:
+ return principal()
+
+ @fastapi_app.post("/organizations/{organization_id}/api-token")
+ async def api_token(
+ _: Organization = Depends(org_auth_service.get_current_org_with_api_token),
+ ) -> dict[str, str | None] | None:
+ return principal()
+
+ @fastapi_app.post("/caller")
+ async def caller(
+ _: CallerContext = Depends(org_auth_service.get_current_caller_context),
+ ) -> dict[str, str | None] | None:
+ return principal()
+
+ @fastapi_app.post("/authentication")
+ async def authentication(
+ _: Organization = Depends(org_auth_service.get_current_org_with_authentication),
+ ) -> dict[str, str | None] | None:
+ return principal()
+
+ @fastapi_app.post("/org-and-user")
+ async def org_and_user(
+ _: Organization = Depends(org_auth_service.get_current_org),
+ __: str | None = Depends(org_auth_service.get_current_user_id_or_none),
+ ) -> dict[str, str | None] | None:
+ return principal()
+
+ @fastapi_app.post("/anonymous")
+ async def anonymous() -> dict[str, str | None] | None:
+ return principal()
+
+ @fastapi_app.websocket("/stream")
+ async def stream(websocket: WebSocket, apikey: str | None = None, token: str | None = None) -> None:
+ await streaming_auth(apikey, token, websocket)
+ await websocket.send_json(principal())
+ await websocket.close()
+
+ return fastapi_app
+
+
+def raw_request_log(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
+ log = MagicMock()
+ monkeypatch.setattr(request_logging, "LOG", log)
+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS", True)
+ return log
+
+
+def _logged_principals(log: MagicMock) -> list[dict[str, object]]:
+ assert all(call.args == ("api.raw_request",) for call in log.info.call_args_list)
+ return [
+ {field: call.kwargs[field] for field in ("organization_id", *_PRINCIPAL_FIELDS)}
+ for call in log.info.call_args_list
+ ]
+
+
+def _use_bearer_authentication(monkeypatch: pytest.MonkeyPatch, organization: Organization | None) -> None:
+ """Configure bearer authentication for ``organization``, or none at all as in a self-hosted deployment."""
+
+ async def authenticate(token: str, attribution_header: str | None = None) -> Organization | None:
+ return None if token == "stale-bearer" else organization
+
+ async def authenticate_user(token: str) -> str | None:
+ return None if token == "stale-bearer" else f"user-of-{token}"
+
+ configured = organization is not None
+ monkeypatch.setattr(org_auth_service.app, "authentication_function", authenticate if configured else None)
+ monkeypatch.setattr(org_auth_service.app, "authenticate_user_function", authenticate_user if configured else None)
+
+
+async def _principal_credentials(
+ monkeypatch: pytest.MonkeyPatch,
+ key_type: str | None,
+ authorization: str | None,
+) -> tuple[dict[str, str], str, _StubIdentityProvider]:
+ headers = {"Authorization": authorization} if authorization else {}
+ if key_type == "api":
+ headers["x-api-key"], _, repository = await _mint_api_token(monkeypatch)
+ elif key_type == "ui_session":
+ headers["x-api-key"], db, repository = await _mint_ui_session_token(monkeypatch)
+ monkeypatch.setattr(org_auth_service.app, "DATABASE", db)
+ else:
+ repository = _FakeOrganizationsRepository(_make_org("org-bearer-only"))
+ _use_bearer_authentication(monkeypatch, None if key_type else repository.organization)
+ identity_provider = _StubIdentityProvider()
+ monkeypatch.setattr(org_auth_service.app, "AGENT_FUNCTION", identity_provider)
+ return headers, repository.organization.organization_id, identity_provider
+
+
+ ("key_type", "authorization", "expected"),
+ [
+ ("api", None, {"auth_kind": "api_key", **_NO_USER}),
+ ("ui_session", None, {"auth_kind": "ui_session", **_NO_USER}),
+ ("api", "Bearer admin-bearer", {"auth_kind": "api_key_and_bearer", **_ADMIN}),
+ ("ui_session", "Bearer member-bearer", {"auth_kind": "ui_session_and_bearer", **_MEMBER}),
+ ("ui_session", "Bearer roleless-bearer", {"auth_kind": "ui_session_and_bearer", **_ROLELESS}),
+ ("ui_session", "Bearer stale-bearer", {"auth_kind": "ui_session", **_NO_USER}),
+ ("ui_session", "Bearer bearer-of-another-organization", {"auth_kind": "ui_session", **_NO_USER}),
+ ("ui_session", "Bearer broken-bearer", {"auth_kind": "ui_session", **_NO_USER}),
+ ("api", "Basic dXNlcjpwYXNzd29yZA==", {"auth_kind": "api_key", **_NO_USER}),
+ (None, "Bearer admin-bearer", {"auth_kind": "bearer", **_ADMIN}),
+ (None, "Bearer roleless-bearer", {"auth_kind": "bearer", **_ROLELESS}),
+ (None, "Bearer bearer-of-another-organization", {"auth_kind": "bearer", **_NO_USER}),
+ ],
+ ids=[
+ "api-key-only",
+ "ui-session-only",
+ "api-key-and-valid-bearer",
+ "ui-session-and-valid-bearer",
+ "ui-session-and-bearer-without-a-role-claim",
+ "ui-session-and-stale-bearer",
+ "ui-session-and-bearer-of-another-organization",
+ "ui-session-and-failing-identity-lookup",
+ "api-key-and-basic-authorization",
+ "bearer-only-with-a-role-claim",
+ "bearer-only-without-a-role-claim",
+ "bearer-only-without-a-proven-user",
+ ],
+)
+async def test_raw_request_record_carries_the_request_principal(
+ monkeypatch: pytest.MonkeyPatch,
+ raw_request_log: MagicMock,
+ key_type: str | None,
+ authorization: str | None,
+ expected: dict[str, str | None],
+) -> None:
+ headers, organization_id, _ = await _principal_credentials(monkeypatch, key_type, authorization)
+
+ with TestClient(_principal_app()) as client:
+ response = client.post("/org", headers=headers)
+
+ principal = {"organization_id": organization_id, **expected}
+ assert response.status_code == 200
+ assert response.json() == principal
+ assert _logged_principals(raw_request_log) == [principal]
+
+
+ ("path", "key_type", "authorization", "expected"),
+ [
+ (path, key_type, authorization, expected)
+ for path in ("/org", "/credential", "/organizations/{organization_id}/api-token", "/caller", "/authentication")
+ for key_type, authorization, expected in (
+ ("api", "Bearer admin-bearer", {"auth_kind": "api_key_and_bearer", **_ADMIN}),
+ ("api", "Bearer stale-bearer", {"auth_kind": "api_key", **_NO_USER}),
+ (None, "Bearer admin-bearer", {"auth_kind": "bearer", **_ADMIN}),
+ )
+ if key_type is None or path != "/authentication"
+ ]
+ + [
+ (path, "ui_session", "Bearer member-bearer", {"auth_kind": "ui_session_and_bearer", **_MEMBER})
+ for path in ("/credential", "/caller")
+ ],
+)
+async def test_every_org_auth_dependency_resolves_the_principal_without_changing_the_outcome(
+ monkeypatch: pytest.MonkeyPatch,
+ raw_request_log: MagicMock,
+ path: str,
+ key_type: str | None,
+ authorization: str,
+ expected: dict[str, str | None],
+) -> None:
+ # The base agent function, which the stub inherits, lets credential routes accept a ui_session token.
+ headers, organization_id, _ = await _principal_credentials(monkeypatch, key_type, authorization)
+ if authorization == "Bearer stale-bearer":
+ # A deployment with bearer authentication configured rejects the stale bearer and falls back to the key.
+ _use_bearer_authentication(monkeypatch, _make_org(organization_id))
+
+ with TestClient(_principal_app()) as client:
+ response = client.post(path.format(organization_id=organization_id), headers=headers)
+
+ principal = {"organization_id": organization_id, **expected}
+ assert response.status_code == 200
+ assert response.json() == principal
+ assert _logged_principals(raw_request_log) == [principal]
+
+
+async def test_two_users_behind_one_key_each_get_their_own_principal_across_a_cache_hit(
+ monkeypatch: pytest.MonkeyPatch,
+ raw_request_log: MagicMock,
+) -> None:
+ headers, organization_id, _ = await _principal_credentials(monkeypatch, "api", None)
+ repository = org_auth_service.app.DATABASE.organizations
+
+ with TestClient(_principal_app()) as client:
+ responses = [
+ client.post("/org", headers={**headers, "Authorization": f"Bearer {bearer}"})
+ for bearer in ("admin-bearer", "member-bearer")
+ ]
+
+ assert repository.validation_calls == 1, "the second request must be served from the key cache"
+ principals = [
+ {"organization_id": organization_id, "auth_kind": "api_key_and_bearer", **user} for user in (_ADMIN, _MEMBER)
+ ]
+ assert [response.json() for response in responses] == principals
+ assert _logged_principals(raw_request_log) == principals
+
+
+async def test_two_auth_dependencies_on_one_route_resolve_the_bearer_once(
+ monkeypatch: pytest.MonkeyPatch,
+ raw_request_log: MagicMock,
+) -> None:
+ headers, organization_id, identity_provider = await _principal_credentials(
+ monkeypatch, "api", "Bearer admin-bearer"
+ )
+ _use_bearer_authentication(monkeypatch, _make_org(organization_id))
+
+ with TestClient(_principal_app()) as client:
+ response = client.post("/org-and-user", headers=headers)
+
+ assert response.json() == {"organization_id": organization_id, "auth_kind": "api_key_and_bearer", **_ADMIN}
+ assert identity_provider.resolved == [("admin-bearer", organization_id)]
+
+
+async def test_caller_context_with_a_bearer_alone_resolves_the_identity_once(
+ monkeypatch: pytest.MonkeyPatch,
+ raw_request_log: MagicMock,
+) -> None:
+ # The org and user helpers run in gathered tasks; the identity lookup must not be paid once per task.
+ headers, organization_id, identity_provider = await _principal_credentials(monkeypatch, None, "Bearer admin-bearer")
+
+ with TestClient(_principal_app()) as client:
+ response = client.post("/caller", headers=headers)
+
+ assert response.json() == {"organization_id": organization_id, "auth_kind": "bearer", **_ADMIN}
+ assert identity_provider.resolved == [("admin-bearer", organization_id)]
+
+
+async def test_a_hanging_identity_lookup_cannot_hold_up_an_authenticated_request(
+ monkeypatch: pytest.MonkeyPatch,
+ raw_request_log: MagicMock,
+) -> None:
+ headers, organization_id, identity_provider = await _principal_credentials(
+ monkeypatch, "api", "Bearer admin-bearer"
+ )
+
+ async def hang(bearer_token: str, organization_id: str) -> BearerIdentity | None:
+ await asyncio.Event().wait()
+ return None
+
+ monkeypatch.setattr(identity_provider, "resolve_bearer_identity", hang)
+ monkeypatch.setattr(request_principal, "BEARER_IDENTITY_TIMEOUT_SECONDS", 0.05)
+
+ with TestClient(_principal_app()) as client:
+ response = client.post("/org", headers=headers)
+
+ assert response.status_code == 200
+ assert response.json() == {"organization_id": organization_id, "auth_kind": "api_key", **_NO_USER}
+
+
+async def test_websocket_in_body_auth_resolves_the_principal(monkeypatch: pytest.MonkeyPatch) -> None:
+ headers, organization_id, _ = await _principal_credentials(monkeypatch, "api", None)
+
+ with (
+ TestClient(_principal_app()) as client,
+ client.websocket_connect(f"/stream?apikey={headers['x-api-key']}&token=Bearer%20member-bearer") as websocket,
+ ):
+ principal = websocket.receive_json()
+
+ assert principal == {"organization_id": organization_id, "auth_kind": "api_key_and_bearer", **_MEMBER}
+
+
+async def test_principal_does_not_leak_into_a_later_unauthenticated_request(
+ monkeypatch: pytest.MonkeyPatch,
+ raw_request_log: MagicMock,
+) -> None:
+ headers, _, _ = await _principal_credentials(monkeypatch, "api", "Bearer admin-bearer")
+
+ with TestClient(_principal_app()) as client:
+ client.post("/org", headers=headers)
+ response = client.post("/anonymous")
+
+ assert response.json() is None
+ assert not set(_PRINCIPAL_FIELDS) & set(raw_request_log.info.call_args.kwargs)
diff --git a/tests/unit/test_script_service_block_outcome.py b/tests/unit/test_script_service_block_outcome.py
new file mode 100644
index 000000000..b0c7494e2
--- /dev/null
+++ b/tests/unit/test_script_service_block_outcome.py
@@ -0,0 +1,131 @@
+"""Outcome records from cached blocks and loops."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext
+from skyvern.forge.sdk.workflow.models.block import ForLoopBlock
+from skyvern.schemas.workflows import BlockStatus
+from skyvern.services import script_service
+from tests.unit.conftest import make_block_output_parameter
+
+
+ ("complete_if_empty", "expected_status", "expected_reason"),
+ [
+ (True, BlockStatus.completed, None),
+ (False, BlockStatus.terminated, "No iterable value found for the loop block"),
+ ],
+)
+async def test_cached_loop_with_no_values_records_its_outcome(
+ monkeypatch: pytest.MonkeyPatch,
+ complete_if_empty: bool,
+ expected_status: BlockStatus,
+ expected_reason: str | None,
+) -> None:
+ context = WorkflowRunContext(
+ workflow_title="test",
+ workflow_id="wf",
+ workflow_permanent_id="wpid",
+ workflow_run_id="wr_cached",
+ aws_client=AsyncMock(),
+ )
+ run_context = skyvern_context.SkyvernContext(
+ organization_id="o_cached", workflow_id="wf", workflow_run_id="wr_cached", script_mode=True
+ )
+ monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=context))
+ monkeypatch.setattr(
+ script_service, "_create_workflow_block_run_and_task", AsyncMock(return_value=("wrb_rows", None, None))
+ )
+ monkeypatch.setattr(
+ script_service,
+ "_validate_and_get_output_parameter",
+ AsyncMock(
+ return_value=script_service.BlockValidationOutput(
+ context=run_context,
+ label="rows",
+ output_parameter=make_block_output_parameter("rows_output"),
+ input_parameters=[],
+ workflow=MagicMock(),
+ workflow_id="wf",
+ workflow_run_id="wr_cached",
+ organization_id="o_cached",
+ )
+ ),
+ )
+ monkeypatch.setattr(ForLoopBlock, "get_values_from_loop_variable_reference", AsyncMock(return_value=[]))
+
+ async def drain() -> None:
+ async for _ in script_service.loop([], complete_if_empty=complete_if_empty, label="rows"):
+ pass
+
+ if complete_if_empty:
+ await drain()
+ else:
+ with pytest.raises(Exception, match="No iterable value"):
+ await drain()
+
+ assert context.get_block_outcome("rows") == BlockOutcome(
+ status=expected_status, error_codes=[], failure_reason=expected_reason
+ )
+
+
+async def test_cached_outcome_survives_output_parameter_failure(monkeypatch: pytest.MonkeyPatch) -> None:
+ context = WorkflowRunContext(
+ workflow_title="test",
+ workflow_id="wf",
+ workflow_permanent_id="wpid",
+ workflow_run_id="wr_cached",
+ aws_client=AsyncMock(),
+ )
+ run_context = skyvern_context.SkyvernContext(
+ organization_id="o_cached", workflow_id="wf", workflow_run_id="wr_cached", script_mode=True
+ )
+ monkeypatch.setattr(script_service.skyvern_context, "current", lambda: run_context)
+ monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=context))
+ monkeypatch.setattr(
+ app.DATABASE.observer,
+ "update_workflow_run_block",
+ AsyncMock(return_value=SimpleNamespace(label="cached", error_codes=["picked"])),
+ )
+ monkeypatch.setattr(
+ script_service, "_record_output_parameter_value", AsyncMock(side_effect=RuntimeError("Output unavailable"))
+ )
+
+ await script_service._update_workflow_block(
+ "wrb_cached", BlockStatus.failed, label="cached", failure_reason="Failed", error_codes=["picked"]
+ )
+
+ assert context.get_block_outcome("cached") == BlockOutcome(
+ status=BlockStatus.failed, error_codes=["picked"], failure_reason="Failed"
+ )
+
+
+async def test_outcome_record_failure_does_not_skip_cached_output(monkeypatch: pytest.MonkeyPatch) -> None:
+ run_context = skyvern_context.SkyvernContext(
+ organization_id="o_cached", workflow_id="wf", workflow_run_id="wr_cached", script_mode=True
+ )
+ monkeypatch.setattr(script_service.skyvern_context, "current", lambda: run_context)
+ workflow_context = MagicMock()
+ workflow_context.record_block_outcome.side_effect = RuntimeError("Outcome unavailable")
+ monkeypatch.setattr(
+ app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=workflow_context)
+ )
+ monkeypatch.setattr(
+ app.DATABASE.observer,
+ "update_workflow_run_block",
+ AsyncMock(return_value=SimpleNamespace(label="cached", error_codes=[])),
+ )
+ record_output = AsyncMock()
+ monkeypatch.setattr(script_service, "_record_output_parameter_value", record_output)
+
+ await script_service._update_workflow_block("wrb_cached", BlockStatus.completed, label="cached")
+
+ record_output.assert_awaited_once()
diff --git a/tests/unit/test_script_service_fallback_engine.py b/tests/unit/test_script_service_fallback_engine.py
index 0703013af..b7bbdf4d8 100644
--- a/tests/unit/test_script_service_fallback_engine.py
+++ b/tests/unit/test_script_service_fallback_engine.py
@@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from skyvern.config import settings
+from skyvern.errors.errors import UserDefinedError
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.forge.sdk.workflow.models.block import (
@@ -146,6 +147,63 @@ async def test_fallback_keeps_default_engine_when_block_missing_from_definition(
assert _fallback_task_block(app).engine == RunEngine.skyvern_v1
+async def test_failed_cached_task_carries_detected_codes_to_block_update() -> None:
+ workflow = _make_workflow([_make_task_block("my_block")])
+ app = _make_app(workflow)
+ app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=SimpleNamespace(ai_fallback=False))
+ app.DATABASE.tasks.get_task = AsyncMock(return_value=SimpleNamespace(errors=[]))
+ app.DATABASE.tasks.update_task = AsyncMock()
+ error = UserDefinedError(error_code="blocked", reasoning="Blocked", confidence_float=1.0)
+ update_block = AsyncMock()
+
+ with (
+ patch(f"{MODULE}.app", app),
+ patch(f"{MODULE}.skyvern_context.current", return_value=_make_context()),
+ patch(f"{MODULE}._detect_user_defined_errors", new=AsyncMock(return_value=[error])),
+ patch(f"{MODULE}._update_workflow_block", update_block),
+ ):
+ await script_service._fallback_to_ai_run(
+ block_type=BlockType.NAVIGATION,
+ cache_key="my_block",
+ error_code_mapping={"blocked": "Blocked"},
+ error=RuntimeError("Script failed"),
+ workflow_run_block_id="wrb_test",
+ )
+
+ assert update_block.await_args.kwargs["error_codes"] == ["blocked"]
+
+
+async def test_ai_fallback_carries_refreshed_task_codes_to_block_update() -> None:
+ workflow = _make_workflow([_make_task_block("my_block")])
+ app = _make_app(workflow)
+ app.DATABASE.tasks.get_task = AsyncMock(
+ side_effect=[
+ SimpleNamespace(url="https://example.com", errors=[]),
+ SimpleNamespace(
+ status=TaskStatus.failed,
+ failure_reason="Fallback identified the failure",
+ errors=[{"error_code": "picked", "reasoning": "Matched"}],
+ ),
+ ]
+ )
+ update_block = AsyncMock()
+
+ with (
+ patch(f"{MODULE}.app", app),
+ patch(f"{MODULE}.skyvern_context.current", return_value=_make_context()),
+ patch(f"{MODULE}._update_workflow_block", update_block),
+ ):
+ await script_service._fallback_to_ai_run(
+ block_type=BlockType.NAVIGATION,
+ cache_key="my_block",
+ workflow_run_block_id="wrb_test",
+ )
+
+ assert update_block.await_args.kwargs["error_codes"] == ["picked"]
+
+
@pytest.mark.asyncio
async def test_fallback_fails_open_to_default_when_engine_lookup_raises() -> None:
workflow = _make_workflow([_make_task_block("my_block", engine=RunEngine.skyvern_v3)])
diff --git a/tests/unit/test_taskv3_goal_composition.py b/tests/unit/test_taskv3_goal_composition.py
index ad34eeaab..6305d73d6 100644
--- a/tests/unit/test_taskv3_goal_composition.py
+++ b/tests/unit/test_taskv3_goal_composition.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import ast
import itertools
import json
+from dataclasses import replace
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
@@ -18,6 +19,7 @@ from skyvern.forge.sdk.workflow.page_derived_templates import OPEN, PageDerivedR
from skyvern.forge.taskv3.goal_composition import (
MAX_HANDOFF_LABEL_CHARS,
PAGE_DATA_NOTE,
+ CodeProgressRecord,
GoalDirectives,
compose_goal,
present_page_derived,
@@ -337,6 +339,28 @@ def test_the_page_data_note_sits_between_the_criteria_and_the_framing_only_when_
assert PAGE_DATA_NOTE not in compose_goal("Apply", GoalDirectives(framing="FRAMING"))
+def test_the_code_outline_is_one_labelled_section_after_everything_else_and_only_when_given() -> None:
+ record = CodeProgressRecord(before=("open", "search"), failed_step="click row", failed_line=4, after=("save",))
+ directives = GoalDirectives(complete_criterion="the form is sent", framing="FRAMING", block_context_section="CTX")
+
+ goal = compose_goal("Apply", replace(directives, code_progress=record))
+ lines = goal.split("\n")
+
+ assert goal.startswith(compose_goal("Apply", directives) + "\n\n")
+ assert goal.count("Code outline") == 1
+ assert "not steps to perform" in goal
+ assert [line for line in lines if line.startswith("- Earlier in the code: ")] == [
+ "- Earlier in the code: open",
+ "- Earlier in the code: search",
+ ]
+ assert [line for line in lines if line.startswith("- Raised an error at line ")] == [
+ "- Raised an error at line 4: click row"
+ ]
+ assert [line for line in lines if line.startswith("- Later in the code: ")] == ["- Later in the code: save"]
+ assert lines[-1] == "- Later in the code: save"
+ assert "Code outline" not in compose_goal("Apply", directives)
+
+
@pytest.mark.parametrize(
("render", "row", "presentation", "qualifier"),
[
diff --git a/tests/unit/test_taskv3_loop.py b/tests/unit/test_taskv3_loop.py
index 7c81c127d..bc41f0c44 100644
--- a/tests/unit/test_taskv3_loop.py
+++ b/tests/unit/test_taskv3_loop.py
@@ -63,6 +63,8 @@ from skyvern.forge.taskv3.loop import (
PERCEPTION_STALL_TERMINATE_AFTER,
PROGRESS_LEDGER_SHADOW_EVENT,
PROGRESS_LEDGER_WINDOW,
+ TOKEN_BUDGET_EXTENDED_EVENT,
+ TOKEN_BUDGET_EXTENSION_REFUSED_EVENT,
UNCHARGED_REFUSAL_GRACE,
VERDICT_URL_MAX_CHARS,
ActivityRecency,
@@ -104,8 +106,10 @@ class _ScriptedCaller:
script: list[list[tuple[str, dict[str, Any]]]],
texts: list[str] | None = None,
reasoning_contents: list[str | None] | None = None,
+ turn_tokens: int = 15,
) -> None:
self._script = script
+ self._turn_tokens = turn_tokens
# Per-turn assistant text, indexed like `script`; falls back to a fixed placeholder so
# existing callers that don't care about the text still get a non-empty one.
self._texts = texts
@@ -167,7 +171,10 @@ class _ScriptedCaller:
{"id": f"call_{i}", "type": "function", "function": {"name": name, "arguments": json.dumps(args)}}
for i, (name, args) in enumerate(turn)
]
- return {"choices": [{"message": message}], "usage": {"prompt_tokens": 10, "completion_tokens": 5}}
+ return {
+ "choices": [{"message": message}],
+ "usage": {"prompt_tokens": self._turn_tokens - 5, "completion_tokens": 5},
+ }
def _recording_tool(
@@ -240,9 +247,10 @@ async def _run(
*,
texts: list[str] | None = None,
reasoning_contents: list[str | None] | None = None,
+ turn_tokens: int = 15,
**kwargs: Any,
):
- caller = _ScriptedCaller(script, texts=texts, reasoning_contents=reasoning_contents)
+ caller = _ScriptedCaller(script, texts=texts, reasoning_contents=reasoning_contents, turn_tokens=turn_tokens)
defaults = {"max_turns": 20, "max_tool_calls": 100}
defaults.update(kwargs)
outcome = await run_agent_tool_loop(
@@ -3427,6 +3435,346 @@ async def test_extension_past_the_token_ceiling_reports_the_clamp() -> None:
assert clamped[0]["step_cap"] == base_cap + base_cap // 2
+async def test_a_token_trip_without_page_change_evidence_is_not_extended() -> None:
+ # A perception spiral re-reading an unchanged page is what the token backstop exists for: with the
+ # step budget unspent, it must still trip exactly as before the token-side extension.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ script = [[("click", {"selector": "#a"})]] + [[("observe", {})] for _ in range(8)]
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [_perception_tool("observe", "the same page"), _recording_tool("click", clicks, billable=True)],
+ max_action_steps=24,
+ max_tokens=75,
+ backstops_for_cap=taskv3_runaway_backstops,
+ )
+ assert outcome.status == "budget_exhausted"
+ assert outcome.cap_trip == "max_tokens (75) reached"
+ assert not [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT]
+ refused = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENSION_REFUSED_EVENT]
+ assert [(entry["guard"], entry["gate_reason"]) for entry in refused] == [
+ ("max_tokens", "no_recent_page_change_evidence")
+ ]
+
+
+async def test_a_token_grant_must_be_re_earned_on_new_page_change_evidence() -> None:
+ # Evidence recency is counted in action rounds, and re-reading a page advances none, so without this a
+ # single page change would fund every later grant of a run that has stopped acting.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for i in range(3):
+ script.append([("observe", {})])
+ script.append([("click", {"selector": f"#a{i}"})])
+ script += [[("observe", {})] for _ in range(8)]
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [_perception_tool("observe", ["page 1", "page 2", "page 3", "page 4"]), _billable_tool("click", clicks)],
+ max_action_steps=24,
+ max_tokens=120,
+ backstops_for_cap=lambda cap: (1000, 1000, cap * 5),
+ )
+ assert [entry["max_tokens"] for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT] == [180]
+ refused = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENSION_REFUSED_EVENT]
+ assert [entry["gate_reason"] for entry in refused] == ["no_recent_page_change_evidence"]
+ assert outcome.cap_trip == "max_tokens (180) reached"
+
+
+_AFTER_THE_FIRST_GRANT: dict[str, list[list[tuple[str, dict[str, Any]]]]] = {
+ "zero_actions": [[("observe", {})] for _ in range(8)],
+ "one_hover_per_six_rereads": ([[("hover", {"selector": "#logo"})]] + [[("observe", {})] for _ in range(6)]) * 2,
+ "identical_submit_retries": [[("click", {"selector": "#submit"}), ("observe", {})] for _ in range(8)],
+ # A new action, but the only fresh read came before it: the change was not the action's.
+ "new_action_after_the_read": [[("observe", {})], [("observe", {})], [("hover", {"selector": "#menu"})]]
+ + [[("wait", {})] for _ in range(6)],
+}
+
+
+ ("pattern", "reason"),
+ [
+ ("zero_actions", "no_new_action_since_last_token_grant"),
+ ("one_hover_per_six_rereads", "no_new_action_since_last_token_grant"),
+ ("identical_submit_retries", "no_new_action_since_last_token_grant"),
+ ("new_action_after_the_read", "no_page_change_after_the_new_action"),
+ ],
+)
+async def test_a_spiral_on_ever_changing_content_gets_no_second_token_grant(pattern: str, reason: str) -> None:
+ # A clock or rotating text makes every re-read fresh evidence, so a renewal needs an action the run has not
+ # already made, followed by a page change. Repeating one action or reading before acting does not count.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("observe", {})]]
+ for action in (("click", "#a0"), ("hover", "#logo"), ("click", "#submit")):
+ script.append([(action[0], {"selector": action[1]})])
+ script.append([("observe", {})])
+ script += _AFTER_THE_FIRST_GRANT[pattern]
+ tools = [
+ _perception_tool("observe", [f"clock at 10:{i:02d}" for i in range(60)]),
+ _billable_tool("click", clicks),
+ _billable_tool("hover", clicks),
+ _recording_tool("wait", clicks),
+ ]
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ tools,
+ max_action_steps=24,
+ max_turns=100,
+ max_tokens=120,
+ backstops_for_cap=lambda cap: (1000, 1000, cap * 5),
+ )
+ assert [entry["max_tokens"] for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT] == [180]
+ refused = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENSION_REFUSED_EVENT]
+ assert [entry["gate_reason"] for entry in refused] == [reason]
+ assert outcome.cap_trip == "max_tokens (180) reached"
+
+
+async def test_an_action_that_failed_before_counts_as_new_when_it_first_succeeds() -> None:
+ # Only a successful action is "made": a field typed once, rejected, and typed again renews the grant.
+ typed: list[tuple[str, dict[str, Any]]] = []
+ attempts: dict[str, int] = {}
+
+ async def flaky_type(args: dict[str, Any]) -> ToolResult:
+ typed.append(("type", args))
+ attempts[args["selector"]] = attempts.get(args["selector"], 0) + 1
+ return ToolResult.error("field rejected") if attempts[args["selector"]] == 1 else ToolResult.ok("typed")
+
+ type_tool = ToolSpec(
+ name="type",
+ description="type",
+ parameters={"type": "object", "properties": {}},
+ handler=flaky_type,
+ billable=True,
+ )
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("observe", {})]]
+ for i in range(3):
+ script.append([("click", {"selector": f"#a{i}"})])
+ script.append([("observe", {})])
+ script += [[("type", {"selector": "#f"})], [("observe", {})], [("type", {"selector": "#f"})]]
+ script += [[("observe", {})] for _ in range(8)]
+ with capture_logs() as logs:
+ await _run(
+ script,
+ [_perception_tool("observe", [f"page {i}" for i in range(60)]), _billable_tool("click", clicks), type_tool],
+ max_action_steps=24,
+ max_turns=100,
+ max_tokens=120,
+ backstops_for_cap=lambda cap: (1000, 1000, cap * 5),
+ )
+ assert [entry["max_tokens"] for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT] == [180, 240]
+
+
+ ("batch", "grants", "reasons"),
+ [
+ # The new action and the fresh read share one action round; the read still came after the action.
+ (
+ [("type", {"selector": "#new", "text": "x"}), ("observe", {})],
+ [180, 240],
+ ["no_recent_page_change_evidence"],
+ ),
+ ([("observe", {}), ("hover", {"selector": "#new"})], [180], ["no_page_change_after_the_new_action"]),
+ # The action's own result reports the page changed: that evidence is the action's, so it follows it.
+ ([("advance", {"selector": "#new"})], [180, 240], ["no_recent_page_change_evidence"]),
+ ],
+)
+async def test_a_batched_new_action_is_ordered_against_its_evidence_within_the_round(
+ batch: list[tuple[str, dict[str, Any]]], grants: list[int], reasons: list[str]
+) -> None:
+ sink: list[tuple[str, dict[str, Any]]] = []
+ script: list[list[tuple[str, dict[str, Any]]]] = [[("observe", {})]]
+ for action in (("click", "#a0"), ("hover", "#logo"), ("click", "#submit")):
+ script.append([(action[0], {"selector": action[1]})])
+ script.append([("observe", {})])
+ script += [[("wait", {})] for _ in range(4)] + [batch]
+ tools = [
+ _perception_tool("observe", [f"clock at 10:{i:02d}" for i in range(60)]),
+ _billable_tool("click", sink),
+ _billable_tool("hover", sink),
+ _billable_tool("type", sink),
+ _recording_tool("wait", sink),
+ _recording_tool("advance", sink, billable=True, ok_data={"page_state_changed": True}),
+ ]
+ with capture_logs() as logs:
+ await _run(
+ script,
+ tools,
+ max_action_steps=24,
+ max_turns=100,
+ max_tokens=120,
+ backstops_for_cap=lambda cap: (1000, 1000, cap * 5),
+ )
+ assert [entry["max_tokens"] for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT] == grants
+ refused = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENSION_REFUSED_EVENT]
+ assert [entry["gate_reason"] for entry in refused] == reasons
+
+
+async def test_a_token_grant_is_clamped_to_the_pool_and_published_to_the_live_activity_record() -> None:
+ # The org's workflow-run pool bounds the token grant as it bounds a step grant: 30 steps left on a 24-step
+ # cap funds the 30-step token guard, not the 36-step one. The raised guard reaches ActivityRecency at once.
+ activity = ActivityRecency()
+ remaining_at_call: list[int | None] = []
+
+ class _ActivityReadingCaller(_ScriptedCaller):
+ async def call(self, **kwargs: Any) -> dict[str, Any]:
+ remaining_at_call.append(activity.tokens_remaining)
+ return await super().call(**kwargs)
+
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for i in range(10):
+ script.append([("observe", {})])
+ script.append([("click", {"selector": f"#a{i}"})])
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ with capture_logs() as logs:
+ outcome = await run_agent_tool_loop(
+ llm_caller=_ActivityReadingCaller(script, turn_tokens=100_000),
+ system_prompt="sys",
+ user_prompt="goal",
+ tools=[_perception_tool("observe", [f"page {i}" for i in range(40)]), _billable_tool("click", clicks)],
+ max_turns=200,
+ max_tool_calls=500,
+ max_action_steps=24,
+ max_action_steps_ceiling=30,
+ max_tokens=1_500_000,
+ final_turn_token_reserve=MAX_TOKENS_PER_ACTION_STEP,
+ backstops_for_cap=taskv3_runaway_backstops,
+ activity=activity,
+ )
+ granted = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT]
+ assert [(entry["max_tokens"], entry["token_extensions_granted"], entry["base_cap"]) for entry in granted] == [
+ (taskv3_runaway_backstops(30)[2], 1, 24)
+ ]
+ grant_turn = granted[0]["turn"]
+ assert remaining_at_call[grant_turn] == taskv3_runaway_backstops(30)[2] - 100_000 * grant_turn
+ refused = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENSION_REFUSED_EVENT]
+ assert [entry["gate_reason"] for entry in refused] == ["extension_limit_reached"]
+ assert outcome.cap_trip == f"max_tokens ({taskv3_runaway_backstops(30)[2]}) reached"
+
+
+async def test_a_step_grant_after_a_token_grant_does_not_report_a_token_clamp() -> None:
+ # A token grant already raised max_tokens to the extended cap's value, so the later step grant buys no
+ # tokens; that is not the sizing function flattening at its ceiling and must not be reported as one.
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for i in range(12):
+ script.append([("observe", {})])
+ script.append([("click", {"selector": f"#a{i}"})])
+ with capture_logs() as logs:
+ await _run(
+ script,
+ [_perception_tool("observe", [f"page {i}" for i in range(40)]), _billable_tool("click", clicks)],
+ max_action_steps=10,
+ max_turns=60,
+ max_tokens=100,
+ backstops_for_cap=lambda cap: (cap * 6, cap * 25, cap * 50),
+ )
+ token_grants = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT]
+ step_grants = [entry for entry in logs if entry["event"] == ACTION_BUDGET_EXTENDED_EVENT]
+ assert token_grants[0]["max_tokens"] == 750 and token_grants[0]["turn"] < step_grants[0]["turn"]
+ assert step_grants[0]["max_tokens"] == 750
+ assert not [entry for entry in logs if entry.get("log_code") == "taskv3_token_backstop_clamped"]
+
+
+async def test_a_token_trip_on_a_stalled_form_is_not_extended_despite_fresh_content() -> None:
+ # Fresh observe content is evidence, but a form whose invalid-field count has not fallen across a
+ # full progress window is stalled, and the stall veto binds on the token path as on the step path.
+ reads = {"n": 0}
+
+ async def observe_handler(args: dict[str, Any]) -> ToolResult:
+ reads["n"] += 1
+ return ToolResult.ok(f"form, look {reads['n']}", data={"summary": {"invalid_fields": 3}})
+
+ observe = ToolSpec(
+ name="observe",
+ description="observe",
+ parameters={"type": "object", "properties": {}},
+ handler=observe_handler,
+ compactable=True,
+ )
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for i in range(PROGRESS_LEDGER_WINDOW + 1):
+ script.append([("observe", {})])
+ script.append([("click", {"selector": f"#a{i}"})])
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [observe, _recording_tool("click", clicks, billable=True)],
+ max_action_steps=24,
+ max_turns=100,
+ max_tokens=15 * len(script),
+ backstops_for_cap=taskv3_runaway_backstops,
+ )
+ assert outcome.cap_trip == f"max_tokens ({15 * len(script)}) reached"
+ refused = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENSION_REFUSED_EVENT]
+ assert [entry["gate_reason"] for entry in refused] == ["no_net_progress_window"]
+
+
+ ("max_action_steps", "turn_tokens", "reserve", "backstops", "expected_caps", "stop_reason"),
+ [
+ # Each grant funds half the ORIGINAL step cap's worth of tokens, and growth stops where the
+ # step path's would: at ACTION_BUDGET_EXTENSION_MAX_FACTOR times the original cap.
+ (20, 15, 0, lambda cap: (1000, 1000, cap * 5), [150, 200, 250, 300], "extension_limit_reached"),
+ # The real sizing policy clamps at MAX_TOKENS_CEILING, and a grant with nothing left to buy stops.
+ (
+ 64,
+ 500_000,
+ MAX_TOKENS_PER_ACTION_STEP,
+ taskv3_runaway_backstops,
+ [MAX_TOKENS_CEILING],
+ "token_ceiling_reached",
+ ),
+ ],
+)
+async def test_token_budget_extension_repeats_but_stops_at_its_bounds(
+ max_action_steps: int,
+ turn_tokens: int,
+ reserve: int,
+ backstops: Callable[[int], tuple[int, int, int]],
+ expected_caps: list[int],
+ stop_reason: str,
+) -> None:
+ observe = _perception_tool("observe", [f"page {i}" for i in range(1, 60)])
+ clicks: list[tuple[str, dict[str, Any]]] = []
+ script: list[list[tuple[str, dict[str, Any]]]] = []
+ for i in range(30):
+ script.append([("observe", {})])
+ script.append([("click", {"selector": f"#a{i}"})])
+ initial_max_tokens = backstops(max_action_steps)[2]
+ with capture_logs() as logs:
+ outcome, _ = await _run(
+ script,
+ [observe, _recording_tool("click", clicks, billable=True)],
+ max_action_steps=max_action_steps,
+ max_turns=1000,
+ max_tool_calls=1000,
+ max_tokens=initial_max_tokens,
+ turn_tokens=turn_tokens,
+ final_turn_token_reserve=reserve,
+ backstops_for_cap=backstops,
+ )
+ granted = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENDED_EVENT]
+ assert [entry["max_tokens"] for entry in granted] == expected_caps
+ assert max(entry["max_tokens"] for entry in granted) <= min(
+ MAX_TOKENS_CEILING, backstops(max_action_steps * ACTION_BUDGET_EXTENSION_MAX_FACTOR)[2]
+ )
+ refused = [entry for entry in logs if entry["event"] == TOKEN_BUDGET_EXTENSION_REFUSED_EVENT]
+ assert [entry["gate_reason"] for entry in refused] == [stop_reason]
+ assert outcome.cap_trip == f"max_tokens ({expected_caps[-1]}) reached"
+ assert len(clicks) < max_action_steps, "the step budget was never the binding cap"
+
+
def test_budget_extension_gate_credits_the_headroom_the_grant_itself_creates() -> None:
# The runaway guards are functions of the action-step budget, so granting an extension also
# raises them. Judging the grant on PRE-grant headroom refuses extensions the grant itself pays
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index c021a6462..d5327ce07 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -2617,16 +2617,35 @@ async def test_route_error_after_staged_commit_clears_stale_proposal_despite_kee
cancelled=False,
)
restore_mock, _ = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+ commits: list[object] = []
+
+ async def record_commit(*args: object, **kwargs: object) -> None:
+ del args
+ commits.append(kwargs)
+
+ monkeypatch.setattr(workflow_copilot_route, "_commit_staged_workflow", record_commit)
+
+ # Fail the step that runs immediately AFTER the commit, so this turn genuinely reaches canonical
+ # before finalisation breaks. The name says "after staged commit"; this makes the harness mean it.
+ persist_calls: list[int] = []
+ original_persist = workflow_copilot_route._persist_proposed_workflow_state
+
+ async def flaky_persist(*args: object, **kwargs: object) -> object:
+ persist_calls.append(1)
+ if len(persist_calls) == 1:
+ raise raised_error
+ return await original_persist(*args, **kwargs)
+
+ monkeypatch.setattr(workflow_copilot_route, "_persist_proposed_workflow_state", flaky_persist)
+
finalise_results: list[object] = []
original_finalise = workflow_copilot_route._finalise_normal_turn
- async def flaky_finalise(*args: object, **kwargs: object) -> object:
+ async def counting_finalise(*args: object, **kwargs: object) -> object:
finalise_results.append(kwargs["agent_result"])
- if len(finalise_results) == 1:
- raise raised_error
return await original_finalise(*args, **kwargs)
- monkeypatch.setattr(workflow_copilot_route, "_finalise_normal_turn", flaky_finalise)
+ monkeypatch.setattr(workflow_copilot_route, "_finalise_normal_turn", counting_finalise)
response = await workflow_copilot_chat_post(
api_key_request, _make_chat_request(keep_pending_proposal=True), organization
@@ -2638,6 +2657,7 @@ async def test_route_error_after_staged_commit_clears_stale_proposal_despite_kee
await handler(copilot_stream)
restore_mock.assert_not_awaited()
+ assert commits, "the turn must actually commit, or this is not the scenario the name describes"
assert len(finalise_results) == 2
recovered_result = finalise_results[1]
assert recovered_result.clear_proposed_workflow is True
@@ -2648,6 +2668,287 @@ async def test_route_error_after_staged_commit_clears_stale_proposal_despite_kee
)
+def _staged_auto_accept_result() -> AgentResult:
+ return AgentResult(
+ user_response="unused",
+ updated_workflow=None,
+ global_llm_context=None,
+ workflow_yaml=None,
+ workflow_was_persisted=False,
+ clear_proposed_workflow=False,
+ authoring_barred=False,
+ resolved_model=None,
+ has_staged_proposal=True,
+ proposal_disposition="auto_applicable",
+ turn_outcome=None,
+ cancelled=False,
+ )
+
+
+def _staged_chat_and_workflow() -> tuple[SimpleNamespace, SimpleNamespace]:
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow={"workflow_id": "pending-review"},
+ auto_accept=True,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ return chat, original_workflow
+
+
+async def test_route_error_when_the_commit_itself_raises_keeps_the_pending_review(
+ monkeypatch: pytest.MonkeyPatch,
+ api_key_request: MagicMock,
+ copilot_stream: MagicMock,
+ organization: SimpleNamespace,
+) -> None:
+ """The commit was eligible and then failed, so canonical never moved and the card the user has
+ not answered is not stale. Nothing about Turn off here: being eligible to commit was enough to
+ discard it."""
+ captured = install_fake_create(monkeypatch)
+ chat, original_workflow = _staged_chat_and_workflow()
+ agent_result = _staged_auto_accept_result()
+ setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+
+ async def failing_commit(*args: object, **kwargs: object) -> None:
+ del args, kwargs
+ raise RuntimeError("commit blew up")
+
+ monkeypatch.setattr(workflow_copilot_route, "_commit_staged_workflow", failing_commit)
+
+ response = await workflow_copilot_chat_post(
+ api_key_request, _make_chat_request(keep_pending_proposal=True), organization
+ )
+ assert response is captured["sentinel"]
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(copilot_stream)
+
+ update_calls = app.DATABASE.workflow_params.update_workflow_copilot_chat.await_args_list
+ clear_calls = [c for c in update_calls if c.kwargs.get("proposed_workflow") is None]
+ assert not clear_calls, f"a commit that raised moved nothing, so the card must survive, got {update_calls!r}"
+
+
+async def test_route_error_after_a_landed_commit_does_not_tell_the_user_nothing_changed(
+ monkeypatch: pytest.MonkeyPatch,
+ api_key_request: MagicMock,
+ copilot_stream: MagicMock,
+ organization: SimpleNamespace,
+) -> None:
+ """The commit reached canonical and the reply write then failed. The receipt must not say the
+ workflow was untouched: it was, and the user can see it on the canvas."""
+ captured = install_fake_create(monkeypatch)
+ chat, original_workflow = _staged_chat_and_workflow()
+ agent_result = _staged_auto_accept_result()
+ _restore_mock, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+
+ commits: list[object] = []
+
+ async def record_commit(*args: object, **kwargs: object) -> None:
+ del args
+ commits.append(kwargs)
+
+ monkeypatch.setattr(workflow_copilot_route, "_commit_staged_workflow", record_commit)
+ workflow_params.create_workflow_copilot_chat_message = AsyncMock(
+ side_effect=RuntimeError("assistant row write failed after the commit")
+ )
+
+ response = await workflow_copilot_chat_post(api_key_request, _make_chat_request(), organization)
+ assert response is captured["sentinel"]
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(copilot_stream)
+
+ assert commits, "the commit must land, or this is not the scenario"
+ # Positive first: a negative-only assertion would also hold if no frame were ever sent.
+ sent = " ".join(str(c) for c in copilot_stream.send.await_args_list)
+ assert copilot_stream.send.await_args_list, "the user must get a reply at all"
+ assert "The workflow was preserved" in sent, f"the reply must say the commit survived, got {sent!r}"
+ assert "The workflow was not modified" not in sent, (
+ f"the commit reached canonical, so this receipt is false: {sent!r}"
+ )
+
+
+async def test_route_error_after_a_landed_commit_clears_the_stale_card_even_if_auto_accept_went_off(
+ monkeypatch: pytest.MonkeyPatch,
+ api_key_request: MagicMock,
+ copilot_stream: MagicMock,
+ organization: SimpleNamespace,
+) -> None:
+ """The commit reached canonical, then finalisation broke and the user turned auto-accept off.
+ The toggle now says off, but canonical has already moved: the old card is stale and accepting it
+ would overwrite the committed workflow. Only the latch can tell these apart."""
+ captured = install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow={"workflow_id": "stale"},
+ auto_accept=True,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ agent_result = AgentResult(
+ user_response="unused",
+ updated_workflow=None,
+ global_llm_context=None,
+ workflow_yaml=None,
+ workflow_was_persisted=False,
+ clear_proposed_workflow=False,
+ authoring_barred=False,
+ resolved_model=None,
+ has_staged_proposal=True,
+ proposal_disposition="auto_applicable",
+ turn_outcome=None,
+ cancelled=False,
+ )
+ _restore_mock, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+
+ commits: list[object] = []
+
+ async def record_commit(*args: object, **kwargs: object) -> None:
+ del args
+ commits.append(kwargs)
+ # The user clicks Turn off after canonical has already been overwritten.
+ chat.auto_accept = False
+
+ monkeypatch.setattr(workflow_copilot_route, "_commit_staged_workflow", record_commit)
+
+ # The row still reads on until the commit lands; the Turn off is only visible after it.
+ async def read_chat(*args: object, **kwargs: object) -> SimpleNamespace:
+ del args, kwargs
+ return SimpleNamespace(**{**vars(chat), "auto_accept": bool(not commits)})
+
+ workflow_params.get_workflow_copilot_chat_by_id = AsyncMock(side_effect=read_chat)
+
+ persist_calls: list[int] = []
+ original_persist = workflow_copilot_route._persist_proposed_workflow_state
+
+ async def flaky_persist(*args: object, **kwargs: object) -> object:
+ persist_calls.append(1)
+ if len(persist_calls) == 1:
+ raise RuntimeError("clearing the card blew up after the commit landed")
+ return await original_persist(*args, **kwargs)
+
+ monkeypatch.setattr(workflow_copilot_route, "_persist_proposed_workflow_state", flaky_persist)
+
+ response = await workflow_copilot_chat_post(
+ api_key_request, _make_chat_request(keep_pending_proposal=True), organization
+ )
+ assert response is captured["sentinel"]
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(copilot_stream)
+
+ assert commits, "the commit must land, or this is not the scenario"
+ update_calls = app.DATABASE.workflow_params.update_workflow_copilot_chat.await_args_list
+ clear_calls = [c for c in update_calls if c.kwargs.get("proposed_workflow") is None]
+ assert clear_calls, f"a landed commit makes the old card stale whatever the toggle says now, got {update_calls!r}"
+
+
+ "raised_error",
+ [RuntimeError("post-agent route boom"), LLMProviderError("OPENAI_GPT5_5")],
+ ids=["generic-exception-handler", "llm-provider-error-handler"],
+)
+async def test_route_error_after_turn_off_keeps_the_review_the_user_has_not_answered(
+ monkeypatch: pytest.MonkeyPatch,
+ raised_error: BaseException,
+ api_key_request: MagicMock,
+ copilot_stream: MagicMock,
+ organization: SimpleNamespace,
+) -> None:
+ """The recovery finalizer infers whether the staged commit happened from the turn-start snapshot.
+ If the user turned auto-accept off mid-turn, no commit happened, so inferring one clears a
+ proposal they never answered."""
+ captured = install_fake_create(monkeypatch)
+
+ chat = SimpleNamespace(
+ workflow_copilot_chat_id="chat-1",
+ workflow_permanent_id="wpid-1",
+ organization_id="org-1",
+ proposed_workflow={"workflow_id": "pending-review"},
+ auto_accept=True,
+ )
+ original_workflow = SimpleNamespace(
+ workflow_id="wf-canonical",
+ title="Original",
+ description="Original description",
+ workflow_definition=None,
+ )
+ agent_result = AgentResult(
+ user_response="unused",
+ updated_workflow=None,
+ global_llm_context=None,
+ workflow_yaml=None,
+ workflow_was_persisted=False,
+ clear_proposed_workflow=False,
+ authoring_barred=False,
+ resolved_model=None,
+ has_staged_proposal=True,
+ proposal_disposition="auto_applicable",
+ turn_outcome=None,
+ cancelled=False,
+ )
+ _restore_mock, workflow_params = setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
+
+ # The turn reads the row once at its start; the user clicks Turn off while it runs, so every
+ # read after that one sees auto_accept off.
+ disabled_row = SimpleNamespace(**{**vars(chat), "auto_accept": False})
+ reads: list[int] = []
+
+ async def read_chat(*args: object, **kwargs: object) -> SimpleNamespace:
+ del args, kwargs
+ reads.append(1)
+ return chat if len(reads) == 1 else disabled_row
+
+ workflow_params.get_workflow_copilot_chat_by_id = AsyncMock(side_effect=read_chat)
+
+ finalise_results: list[object] = []
+ original_finalise = workflow_copilot_route._finalise_normal_turn
+
+ async def flaky_finalise(*args: object, **kwargs: object) -> object:
+ finalise_results.append(kwargs["agent_result"])
+ if len(finalise_results) == 1:
+ # Fires before the normal finalizer re-reads the row, so nothing has been committed.
+ raise raised_error
+ return await original_finalise(*args, **kwargs)
+
+ monkeypatch.setattr(workflow_copilot_route, "_finalise_normal_turn", flaky_finalise)
+
+ response = await workflow_copilot_chat_post(
+ api_key_request, _make_chat_request(keep_pending_proposal=True), organization
+ )
+ assert response is captured["sentinel"]
+ handler = captured["handler"]
+ assert callable(handler)
+ await handler(copilot_stream)
+
+ assert len(finalise_results) == 2
+ recovered_result = finalise_results[1]
+ assert recovered_result.clear_proposed_workflow is False
+ update_calls = app.DATABASE.workflow_params.update_workflow_copilot_chat.await_args_list
+ clear_calls = [c for c in update_calls if c.kwargs.get("proposed_workflow") is None]
+ assert not clear_calls, (
+ f"a turn the user switched off staged nothing, so the pending review must survive, got {update_calls!r}"
+ )
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"scenario",
diff --git a/tests/unit/webeye/test_browser_session_response.py b/tests/unit/webeye/test_browser_session_response.py
index 8022916f8..ccf813291 100644
--- a/tests/unit/webeye/test_browser_session_response.py
+++ b/tests/unit/webeye/test_browser_session_response.py
@@ -3,6 +3,7 @@ from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from fastapi import HTTPException
from skyvern.forge import app
from skyvern.forge.agent_functions import AgentFunction
@@ -151,6 +152,231 @@ async def test_browser_session_response_bounds_infrastructure_recording_selectio
assert response.recordings == []
+class FakeListing:
+ """A storage listing that returns or raises its outcome once released, or never finishes when held."""
+
+ def __init__(self, outcome: list[FileInfo] | Exception, *, held: bool = False) -> None:
+ self.outcome = outcome
+ self.started = asyncio.Event()
+ self.release = asyncio.Event()
+ if not held:
+ self.release.set()
+ self.in_flight = False
+
+ async def __call__(self, **_kwargs: object) -> list[FileInfo]:
+ self.in_flight = True
+ self.started.set()
+ try:
+ await self.release.wait()
+ finally:
+ self.in_flight = False
+ if isinstance(self.outcome, Exception):
+ raise self.outcome
+ return self.outcome
+
+
+def storage_listing(downloads: FakeListing, recordings: FakeListing) -> MagicMock:
+ storage = MagicMock()
+ storage.get_shared_downloaded_files_in_browser_session = downloads
+ storage.get_shared_recordings_in_browser_session = recordings
+ return storage
+
+
+def completed_session() -> PersistentBrowserSession:
+ now = datetime.now(timezone.utc)
+ return PersistentBrowserSession(
+ persistent_browser_session_id="pbs_123",
+ organization_id="org_123",
+ status="completed",
+ created_at=now,
+ modified_at=now,
+ )
+
+
+DOWNLOAD = FileInfo(url="https://files.example/report.pdf", filename="report.pdf")
+RECORDING = FileInfo(url="https://recordings.example/session.webm", filename="session.webm")
+
+
+def base_agent_function(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The stub app mocks the agent function; these tests need the hooks to pass values through."""
+ monkeypatch.setattr(app, "AGENT_FUNCTION", AgentFunction())
+
+
[email protected]("base_agent_function")
+async def test_browser_session_response_runs_the_downloads_and_recordings_listings_together() -> None:
+ downloads = FakeListing([DOWNLOAD])
+ recordings = FakeListing([RECORDING])
+ # Each listing finishes only once the other has started, so neither can finish unless they overlap.
+ downloads.release, recordings.release = recordings.started, downloads.started
+
+ response = await asyncio.wait_for(
+ BrowserSessionResponse.from_browser_session(
+ completed_session(), storage_listing(downloads, recordings), concurrent_listings=True
+ ),
+ timeout=1,
+ )
+
+ assert response.downloaded_files == [DOWNLOAD]
+ assert response.recordings == [RECORDING]
+
+
[email protected]("base_agent_function")
+async def test_browser_session_response_lists_downloads_then_recordings_by_default() -> None:
+ """The fan-out endpoints gather across sessions already, so a session's two listings must not
+ overlap there: that would double the request's peak pool checkouts for no gain."""
+ downloads = FakeListing([DOWNLOAD], held=True)
+ recordings = FakeListing([RECORDING])
+ building = asyncio.create_task(
+ BrowserSessionResponse.from_browser_session(completed_session(), storage_listing(downloads, recordings))
+ )
+ await asyncio.wait_for(downloads.started.wait(), timeout=1)
+ for _ in range(3):
+ await asyncio.sleep(0)
+
+ assert not recordings.started.is_set()
+
+ downloads.release.set()
+ response = await asyncio.wait_for(building, timeout=1)
+ assert response.downloaded_files == [DOWNLOAD]
+ assert response.recordings == [RECORDING]
+
+
+ ("downloads_error", "raised"),
+ [(RuntimeError("storage unavailable"), RuntimeError), (TimeoutError(), HTTPException)],
+ ids=["error", "strict-timeout"],
+)
+async def test_browser_session_response_stops_the_recordings_listing_when_the_downloads_listing_fails(
+ downloads_error: Exception, raised: type[Exception]
+) -> None:
+ downloads = FakeListing(downloads_error)
+ recordings = FakeListing([RECORDING], held=True)
+ downloads.release = recordings.started
+
+ with pytest.raises(raised):
+ await asyncio.wait_for(
+ BrowserSessionResponse.from_browser_session(
+ completed_session(),
+ storage_listing(downloads, recordings),
+ fail_download_lookup=True,
+ concurrent_listings=True,
+ ),
+ timeout=1,
+ )
+
+ assert recordings.started.is_set()
+ assert not recordings.in_flight
+
+
+async def test_browser_session_response_answers_for_the_downloads_listing_when_both_listings_fail() -> None:
+ """A storage outage fails both listings; the strict lookup's retryable 503 must not lose to the recordings error."""
+ downloads = FakeListing(TimeoutError())
+ recordings = FakeListing(RuntimeError("storage unavailable"))
+ downloads.release = recordings.started
+
+ with pytest.raises(HTTPException) as exc_info:
+ await asyncio.wait_for(
+ BrowserSessionResponse.from_browser_session(
+ completed_session(),
+ storage_listing(downloads, recordings),
+ fail_download_lookup=True,
+ concurrent_listings=True,
+ ),
+ timeout=1,
+ )
+
+ assert exc_info.value.status_code == 503
+
+
+async def test_browser_session_response_stops_both_listings_when_the_request_is_cancelled() -> None:
+ downloads = FakeListing([DOWNLOAD], held=True)
+ recordings = FakeListing([RECORDING], held=True)
+ building = asyncio.create_task(
+ BrowserSessionResponse.from_browser_session(
+ completed_session(), storage_listing(downloads, recordings), concurrent_listings=True
+ )
+ )
+ await asyncio.wait_for(asyncio.gather(downloads.started.wait(), recordings.started.wait()), timeout=1)
+
+ building.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await building
+
+ assert not downloads.in_flight
+ assert not recordings.in_flight
+
+
[email protected]("base_agent_function")
+async def test_browser_session_response_keeps_downloads_when_the_recordings_listing_runs_out_of_time(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The strict lookup answers 503 only for its own listing; a stalled recordings listing is not its failure."""
+ storage = storage_listing(downloads=FakeListing([DOWNLOAD]), recordings=FakeListing([RECORDING], held=True))
+ monkeypatch.setattr(browser_session_schemas, "GET_DOWNLOADED_FILES_TIMEOUT", 0.01)
+
+ response = await asyncio.wait_for(
+ BrowserSessionResponse.from_browser_session(completed_session(), storage, fail_download_lookup=True),
+ timeout=1,
+ )
+
+ assert response.downloaded_files == [DOWNLOAD]
+ assert response.recordings == []
+
+
[email protected]("base_agent_function")
+async def test_browser_session_response_keeps_recordings_when_the_downloads_listing_runs_out_of_time(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ storage = storage_listing(downloads=FakeListing([DOWNLOAD], held=True), recordings=FakeListing([RECORDING]))
+ monkeypatch.setattr(browser_session_schemas, "GET_DOWNLOADED_FILES_TIMEOUT", 0.01)
+
+ response = await asyncio.wait_for(
+ BrowserSessionResponse.from_browser_session(completed_session(), storage), timeout=1
+ )
+
+ assert response.downloaded_files == []
+ assert response.recordings == [RECORDING]
+
+
[email protected]("failing", ["downloads", "recordings"])
+async def test_browser_session_response_surfaces_a_listing_error_that_is_not_a_timeout(failing: str) -> None:
+ error = RuntimeError("storage unavailable")
+ storage = storage_listing(
+ downloads=FakeListing(error if failing == "downloads" else [DOWNLOAD]),
+ recordings=FakeListing(error if failing == "recordings" else [RECORDING]),
+ )
+
+ with pytest.raises(RuntimeError) as exc_info:
+ await BrowserSessionResponse.from_browser_session(completed_session(), storage)
+
+ assert exc_info.value is error
+
+
[email protected]("base_agent_function")
+async def test_browser_session_response_lists_files_newest_first_with_undated_files_last() -> None:
+ older = FileInfo(url="https://files.example/older", modified_at=datetime(2026, 1, 1, tzinfo=timezone.utc))
+ newer = FileInfo(url="https://files.example/newer", modified_at=datetime(2026, 1, 2, tzinfo=timezone.utc))
+ undated = FileInfo(url="https://files.example/undated")
+ storage = storage_listing(
+ downloads=FakeListing([older, undated, newer]), recordings=FakeListing([undated, newer, older])
+ )
+
+ response = await BrowserSessionResponse.from_browser_session(completed_session(), storage)
+
+ assert response.downloaded_files == [newer, older, undated]
+ assert response.recordings == [newer, older, undated]
+
+
def test_no_server_side_row_field_becomes_a_response_field() -> None:
leaked = server_side_row_fields() & set(BrowserSessionResponse.model_fields)
assert leaked == set()
diff --git a/tests/unit/workflow/test_code_block_escalation.py b/tests/unit/workflow/test_code_block_escalation.py
index 7212278d9..f7c020ccb 100644
--- a/tests/unit/workflow/test_code_block_escalation.py
+++ b/tests/unit/workflow/test_code_block_escalation.py
@@ -25,6 +25,7 @@ from skyvern.exceptions import NO_ADDRESS_RECORD_NAV_ERROR_CODE
from skyvern.forge import app
from skyvern.forge.agent_functions import CodeBlockEngineFailure, CodeBlockEngineResult
from skyvern.forge.sdk.api.llm.schema_validator import validate_and_fill_extraction_result, validate_schema
+from skyvern.forge.sdk.copilot.code_block_steps import derive_code_block_steps
from skyvern.forge.sdk.copilot.nav_attribution import (
block_nav_error_codes,
proxy_owns_nav_codes,
@@ -55,6 +56,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
WorkflowParameterType,
)
from skyvern.forge.sdk.workflow.service import _merge_workflow_run_errors
+from skyvern.forge.taskv3.goal_composition import CodeProgressRecord, GoalDirectives, compose_goal
from skyvern.schemas.runs import RunEngine
from skyvern.schemas.self_heal import HealClassification, HealSkipReason, HealStatus
from skyvern.webeye.actions.action_types import ActionType
@@ -1376,12 +1378,12 @@ async def test_prompt_only_heal_fires_without_a_matched_step(
assert result is not None and result.success is True
assert state["execute_step_calls"] == 1
- # No step narrows the goal, so it is the bare block prompt (no MINI_GOAL wrapper).
assert state["create_task_kwargs"]["navigation_goal"] == DEFAULT_PROMPT
+ assert state["execute_step_kwargs"]["recovery_code_progress"] is None
@pytest.mark.asyncio
-async def test_matched_step_narrows_the_goal(
+async def test_matched_step_is_a_record_beside_the_goal_not_part_of_it(
monkeypatch: pytest.MonkeyPatch, ai_fallback_flag: Callable[[str | None], None]
) -> None:
ai_fallback_flag("o_test")
@@ -1393,10 +1395,10 @@ async def test_matched_step_narrows_the_goal(
result = await _heal(block, _make_context(), exc, _recording_page(exc), failing_line=1)
assert result is not None and result.success is True
- goal = state["create_task_kwargs"]["navigation_goal"]
- assert goal != DEFAULT_PROMPT
- assert "click the export button" in goal
- assert DEFAULT_PROMPT in goal
+ assert state["create_task_kwargs"]["navigation_goal"] == DEFAULT_PROMPT
+ assert state["execute_step_kwargs"]["recovery_code_progress"] == CodeProgressRecord(
+ before=(), failed_step="click the export button", failed_line=1, after=()
+ )
@pytest.mark.parametrize("prompt", [None, ""], ids=["absent_goal", "empty_goal"])
@@ -1411,7 +1413,7 @@ def test_steps_alone_never_manufacture_a_heal_goal(prompt: str | None) -> None:
)
context = _make_context()
- assert block._compose_heal_goal(workflow_run_context=context, failing_line=1) == ""
+ assert block._compose_heal_goal(workflow_run_context=context) == ""
def test_failing_goto_heals_toward_its_own_url_not_an_address_in_the_step_outline() -> None:
@@ -1426,18 +1428,6 @@ def test_failing_goto_heals_toward_its_own_url_not_an_address_in_the_step_outlin
assert block._derive_escalation_navigation_url(2, _recording_page(None)) == "https://example.com/b"
-def test_an_authored_goal_is_still_narrowed_by_its_matched_step() -> None:
- block = _make_code_block(
- steps=[CodeBlockStep(description="click the export button", line_start=1, line_end=1)],
- )
- context = _make_context()
-
- goal = block._compose_heal_goal(workflow_run_context=context, failing_line=1)
-
- assert "click the export button" in goal
- assert DEFAULT_PROMPT in goal
-
-
@pytest.mark.asyncio
async def test_failing_static_goto_sets_escalation_url(
monkeypatch: pytest.MonkeyPatch, ai_fallback_flag: Callable[[str | None], None]
@@ -2803,6 +2793,7 @@ async def test_secret_value_never_leaks_into_goal_or_task(
goal = state["create_task_kwargs"]["navigation_goal"]
assert SECRET_VALUE not in goal
assert "*****" in goal
+ assert state["execute_step_kwargs"]["recovery_code_progress"].failed_step == "submit token *****"
assert _string_values(state["create_task_kwargs"]["extracted_information_schema"])
assert "receipt_*****" in state["create_task_kwargs"]["data_extraction_goal"]
for value in state["create_task_kwargs"].values():
@@ -2940,7 +2931,7 @@ async def test_lone_line_start_step_is_matched(
assert result is not None
assert state["execute_step_calls"] == 1
- assert "open the menu" in state["create_task_kwargs"]["navigation_goal"]
+ assert state["execute_step_kwargs"]["recovery_code_progress"].failed_step == "open the menu"
def test_match_step_picks_largest_preceding_start() -> None:
@@ -3045,7 +3036,7 @@ async def test_recovery_block_finalized_to_non_completed_status(
@pytest.mark.asyncio
-async def test_mid_block_failure_composes_remaining_steps(
+async def test_mid_block_failure_keeps_the_goal_verbatim_and_records_the_outline(
monkeypatch: pytest.MonkeyPatch, ai_fallback_flag: Callable[[str | None], None]
) -> None:
ai_fallback_flag("o_test")
@@ -3064,11 +3055,70 @@ async def test_mid_block_failure_composes_remaining_steps(
assert result is not None and result.success is True
goal = state["create_task_kwargs"]["navigation_goal"]
- assert "click the invoices tab" in goal
- assert "Then: download the latest invoice" in goal
- # steps before the failure already ran as code — they must not be re-demanded.
- assert "open the portal" not in goal
- assert DEFAULT_PROMPT in goal
+ assert goal == DEFAULT_PROMPT
+ assert "mini goal" not in goal
+ assert "Then:" not in goal
+ record = state["execute_step_kwargs"]["recovery_code_progress"]
+ assert record == CodeProgressRecord(
+ before=("open the portal",),
+ failed_step="click the invoices tab",
+ failed_line=2,
+ after=("download the latest invoice",),
+ )
+ message = compose_goal(goal, GoalDirectives(code_progress=record))
+ assert message.startswith(DEFAULT_PROMPT + "\n\n")
+ assert message.count("Code outline") == 1
+ assert "- Earlier in the code: open the portal" in message
+ assert "- Raised an error at line 2: click the invoices tab" in message
+ assert "- Later in the code: download the latest invoice" in message
+
+
+ ("failing_line", "expected"),
+ [
+ pytest.param(
+ 3,
+ CodeProgressRecord(
+ before=("Open https://example.com/list",),
+ failed_step='Click "Open row"',
+ failed_line=3,
+ after=('Click "Dismiss"', 'Click "Skip"', 'Click "Export"'),
+ ),
+ id="inside_loop",
+ ),
+ pytest.param(
+ 7,
+ CodeProgressRecord(
+ before=("Open https://example.com/list", 'Click "Open row"', 'Click "Dismiss"'),
+ failed_step='Click "Skip"',
+ failed_line=7,
+ after=('Click "Export"',),
+ ),
+ id="else_branch",
+ ),
+ ],
+)
+def test_outline_is_source_order_and_never_claims_what_ran(failing_line: int, expected: CodeProgressRecord) -> None:
+ code = (
+ 'await page.goto("https://example.com/list")\n'
+ "for row in rows:\n"
+ ' await page.get_by_role("button", name="Open row").click()\n'
+ "if banner:\n"
+ ' await page.get_by_role("button", name="Dismiss").click()\n'
+ "else:\n"
+ ' await page.get_by_role("button", name="Skip").click()\n'
+ 'await page.get_by_role("button", name="Export").click()\n'
+ )
+ block = _make_code_block(
+ code=code, steps=[CodeBlockStep.model_validate(step) for step in derive_code_block_steps(code)]
+ )
+
+ record = block._code_progress_record(workflow_run_context=_make_context(), failing_line=failing_line)
+
+ assert record == expected
+ section = compose_goal(DEFAULT_PROMPT, GoalDirectives(code_progress=record)).removeprefix(DEFAULT_PROMPT).lower()
+ for claim in ("completed", "not run", " ran", "already", "done"):
+ assert claim not in section
@pytest.mark.asyncio
@@ -3089,9 +3139,10 @@ async def test_last_step_failure_keeps_single_step_goal(
result = await _heal(block, _make_context(), exc, _recording_page(exc), failing_line=2)
assert result is not None and result.success is True
- goal = state["create_task_kwargs"]["navigation_goal"]
- assert "download the latest invoice" in goal
- assert "Then:" not in goal
+ assert state["create_task_kwargs"]["navigation_goal"] == DEFAULT_PROMPT
+ assert state["execute_step_kwargs"]["recovery_code_progress"] == CodeProgressRecord(
+ before=("open the portal",), failed_step="download the latest invoice", failed_line=2, after=()
+ )
@pytest.mark.asyncio
@@ -3112,9 +3163,9 @@ async def test_remaining_steps_without_descriptions_are_skipped(
result = await _heal(block, _make_context(), exc, _recording_page(exc), failing_line=1)
assert result is not None and result.success is True
- goal = state["create_task_kwargs"]["navigation_goal"]
- assert "click the invoices tab" in goal
- assert "Then:" not in goal
+ assert state["execute_step_kwargs"]["recovery_code_progress"] == CodeProgressRecord(
+ before=(), failed_step="click the invoices tab", failed_line=1, after=()
+ )
@pytest.mark.asyncio
@@ -3135,10 +3186,10 @@ async def test_remaining_step_descriptions_are_masked(
result = await _heal(block, _make_context(with_secret=True), exc, _recording_page(exc), failing_line=1)
assert result is not None and result.success is True
- goal = state["create_task_kwargs"]["navigation_goal"]
- assert "open the portal" in goal
- assert "Then:" in goal
- assert SECRET_VALUE not in goal
+ record = state["execute_step_kwargs"]["recovery_code_progress"]
+ assert record.after == ("submit token *****",)
+ message = compose_goal(state["create_task_kwargs"]["navigation_goal"], GoalDirectives(code_progress=record))
+ assert SECRET_VALUE not in message
@pytest.mark.asyncio
diff --git a/tests/unit/workflow/test_conditional_branch_evaluation.py b/tests/unit/workflow/test_conditional_branch_evaluation.py
index ff3ff0838..32e8aa3e8 100644
--- a/tests/unit/workflow/test_conditional_branch_evaluation.py
+++ b/tests/unit/workflow/test_conditional_branch_evaluation.py
@@ -1,5 +1,6 @@
"""Tests for prompt-based conditional branch evaluation behavior."""
+import copy
import json
from collections.abc import Iterator
from datetime import UTC, datetime
@@ -13,6 +14,8 @@ from skyvern.exceptions import BranchEvaluationContextTooLargeError, Conditional
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.exceptions import MissingJinjaVariables
from skyvern.forge.sdk.workflow.models.block import (
BranchCondition,
@@ -26,9 +29,11 @@ from skyvern.forge.sdk.workflow.models.block import (
_make_empty_params_explicit,
_neutralize_jinja_delimiters,
)
-from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameter, WorkflowParameterType
+from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowDefinition
from skyvern.schemas.run_enums import RunEngine
from skyvern.schemas.workflows import BlockResult, BlockStatus
+from skyvern.utils.token_counter import count_tokens
from tests.unit.fake_workflow_run_context import FakeWorkflowRunContext
BRANCH_CONTEXT_TOO_LARGE_FAILURE_REASON = (
@@ -359,6 +364,232 @@ async def test_branch_goal_above_reserved_budget_trips_before_extraction() -> No
mock_extraction_cls.assert_not_called()
+def _prompt_conditional(label: str) -> ConditionalBlock:
+ return ConditionalBlock(
+ label=label,
+ output_parameter=_output_parameter(f"{label}_output"),
+ branch_conditions=[
+ BranchCondition(
+ criteria=PromptBranchCriteria(expression="the customer is on the premium plan"),
+ next_block_label=f"{label}_matched",
+ ),
+ BranchCondition(is_default=True, next_block_label=f"{label}_default"),
+ ],
+ )
+
+
+def _run_context(*blocks: ConditionalBlock) -> WorkflowRunContext:
+ now = datetime.now(UTC)
+ return WorkflowRunContext(
+ workflow_title="wf",
+ workflow_id="wf",
+ workflow_permanent_id="wpid",
+ workflow_run_id="wr",
+ aws_client=MagicMock(),
+ workflow=Workflow(
+ workflow_id="wf",
+ organization_id="org",
+ title="wf",
+ workflow_permanent_id="wpid",
+ version=1,
+ is_saved_task=False,
+ workflow_definition=WorkflowDefinition(parameters=[], blocks=list(blocks)),
+ created_at=now,
+ modified_at=now,
+ ),
+ )
+
+
+_EVALUATION_REASONING = "reasoning-marker: the stored plan is premium"
+
+
+async def _execute_prompt_conditional(block: ConditionalBlock, run_context: WorkflowRunContext) -> dict:
+ """Run the block against a real run context; only the model call and the database write are faked."""
+
+ async def _condition_is_true(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202
+ return _extraction_result(
+ self.output_parameter,
+ [{"condition_index": 1, "reasoning": _EVALUATION_REASONING, "result": True}],
+ )
+
+ with (
+ patch.object(ExtractionBlock, "execute", _condition_is_true),
+ patch.object(
+ block_module.app.WORKFLOW_CONTEXT_MANAGER,
+ "get_workflow_run_context",
+ new=MagicMock(return_value=run_context),
+ ),
+ patch.object(
+ block_module.app.DATABASE.workflow_runs,
+ "create_or_update_workflow_run_output_parameter",
+ new=AsyncMock(),
+ ),
+ patch.object(ConditionalBlock, "build_block_result", new_callable=AsyncMock) as mock_build_result,
+ ):
+ await block.execute(workflow_run_id="wr", workflow_run_block_id="wrb", organization_id="org")
+ return mock_build_result.await_args.kwargs
+
+
+ ("output_share_of_budget", "expected_branch", "expected_evaluation_error"),
+ [
+ pytest.param(0.6, "cond_2_matched", None, id="one-copy-fits"),
+ pytest.param(1.2, "cond_2_default", BRANCH_CONTEXT_TOO_LARGE_FAILURE_REASON, id="one-copy-too-large"),
+ ],
+)
+async def test_prior_block_output_counts_once_against_the_branch_context_budget(
+ output_share_of_budget: float, expected_branch: str, expected_evaluation_error: str | None
+) -> None:
+ cond_1, cond_2 = _prompt_conditional("cond_1"), _prompt_conditional("cond_2")
+ run_context = _run_context(cond_1, cond_2)
+ await _execute_prompt_conditional(cond_1, run_context)
+ row = "invoice 1042 paid in full; "
+ tokens_per_row = count_tokens(row * 100) / 100
+ row_count = int(output_share_of_budget * block_module.BRANCH_EVALUATION_GOAL_MAX_TOKENS / tokens_per_row)
+ await run_context.register_output_parameter_value_post_execution(
+ _output_parameter("invoices_output"), {"extracted_information": {"rows": row * row_count}}
+ )
+
+ result = await _execute_prompt_conditional(cond_2, run_context)
+
+ assert result["status"] == BlockStatus.completed
+ assert result["executed_branch_next_block"] == expected_branch
+ assert result["output_parameter_value"].get("evaluation_error") == expected_evaluation_error
+
+
+async def test_branch_context_snapshot_holds_each_output_once_and_no_earlier_evaluation_payload() -> None:
+ cond_1 = _prompt_conditional("cond_1")
+ run_context = _run_context(cond_1, _prompt_conditional("cond_2"))
+ # Present while cond_1 evaluates, so the prompt cond_1 stores embeds it.
+ run_context.values["plan"] = "plan-marker-premium"
+ await _execute_prompt_conditional(cond_1, run_context)
+ # Not a conditional: a field that shares a name with a conditional's debug field is this block's data.
+ lookup_output = {"status": "paid", "evaluations": ["manual review passed"]}
+ for label in ("first_lookup", "second_lookup"):
+ await run_context.register_output_parameter_value_post_execution(
+ _output_parameter(f"{label}_output"), lookup_output
+ )
+ values_before = copy.deepcopy(run_context.values)
+
+ snapshot = BranchEvaluationContext(
+ workflow_run_context=run_context, block_label="cond_2"
+ ).build_llm_safe_context_snapshot()
+
+ # Two different blocks with equal outputs both stay; each is listed once, findable by its label.
+ assert {key: value for key, value in snapshot.items() if "lookup" in key} == {
+ "first_lookup_output": lookup_output,
+ "second_lookup_output": lookup_output,
+ }
+ assert [key for key in snapshot if key.startswith("cond_1")] == ["cond_1_output"]
+ # What a later condition can refer to survives: which branch the earlier conditional took.
+ assert snapshot["cond_1_output"]["branch_taken"] == "cond_1_matched"
+ assert snapshot["cond_1_output"]["branch_index"] == 0
+ assert snapshot["cond_1_output"]["criteria_expression"] == "the customer is on the premium plan"
+ # Its stored prompt, raw response and per-branch detail do not.
+ assert not {"llm_prompt", "llm_response", "evaluations"} & snapshot["cond_1_output"].keys()
+ serialized = json.dumps(snapshot, default=str)
+ assert serialized.count("plan-marker-premium") == 1
+ assert _EVALUATION_REASONING not in serialized
+ # Only the snapshot changes: templates still resolve both names against the untouched run context.
+ assert run_context.values == values_before
+ assert _EVALUATION_REASONING in json.dumps(run_context.values["cond_1"], default=str)
+
+
+async def test_branch_context_snapshot_deduplicates_carried_outputs_and_strips_carried_conditional_debug() -> None:
+ now = datetime.now(UTC)
+ conditional_output = {
+ "branch_taken": "prior_matched",
+ "llm_prompt": "stored prompt",
+ "llm_response": "stored response",
+ "evaluations": [{"reasoning": "stored detail"}],
+ }
+ plain_output = {"status": "paid", "evaluations": ["manual review passed"]}
+ workflow = _run_context(_prompt_conditional("prior")).workflow
+ assert workflow is not None
+ run_context = await WorkflowRunContext.init(
+ aws_client=MagicMock(),
+ organization=Organization(organization_id="org", organization_name="test", created_at=now, modified_at=now),
+ workflow_run_id="wr",
+ workflow_title="wf",
+ workflow_id="wf",
+ workflow_permanent_id="wpid",
+ workflow_parameter_tuples=[],
+ workflow_output_parameters=[],
+ context_parameters=[],
+ secret_parameters=[],
+ block_outputs={"prior": conditional_output, "lookup": plain_output},
+ workflow=workflow,
+ )
+
+ snapshot = BranchEvaluationContext(workflow_run_context=run_context).build_llm_safe_context_snapshot()
+
+ assert run_context.workflow_run_outputs == {}
+ assert run_context.carried_block_labels == {"prior", "lookup"}
+ assert snapshot["prior_output"] == {"branch_taken": "prior_matched"}
+ assert snapshot["lookup_output"] == plain_output
+ assert not {"prior", "lookup"} & snapshot.keys()
+ assert run_context.values["prior"] == conditional_output
+
+
+async def test_branch_context_snapshot_strips_only_recorded_conditional_outputs_and_keeps_declared_keys() -> None:
+ """A conditional's output is known by its recorded output key and alias, not by a label a parameter may
+ share; a declared parameter is the author's whatever its name."""
+ now = datetime.now(UTC)
+
+ def _declare(key: str, value: object) -> None:
+ run_context.parameters[key] = WorkflowParameter(
+ key=key,
+ workflow_parameter_id=f"wp_{key}",
+ workflow_parameter_type=WorkflowParameterType.STRING,
+ workflow_id="wf",
+ created_at=now,
+ modified_at=now,
+ )
+ run_context.values[key] = value
+
+ conditional = _prompt_conditional("cond")
+ conditional.output_parameter = _output_parameter("decision_output")
+ run_context = _run_context(conditional, _prompt_conditional("pending"))
+ _declare("prompt_branch_eval_flag", "yes")
+ # A parameter that shares the not-yet-run conditional's label: author data, kept whole.
+ _declare("pending", {"llm_prompt": "author text", "answer": 1})
+ await _execute_prompt_conditional(conditional, run_context)
+
+ snapshot = BranchEvaluationContext(
+ workflow_run_context=run_context, block_label="next"
+ ).build_llm_safe_context_snapshot()
+
+ assert snapshot["prompt_branch_eval_flag"] == "yes"
+ assert [key for key in snapshot if key.startswith("prompt_branch_eval_")] == ["prompt_branch_eval_flag"]
+ assert snapshot["pending"] == {"llm_prompt": "author text", "answer": 1}
+ assert "decision" not in snapshot
+ assert snapshot["decision_output"]["branch_taken"] == "cond_matched"
+ assert not {"llm_prompt", "llm_response", "evaluations"} & snapshot["decision_output"].keys()
+
+
+async def test_branch_context_snapshot_keeps_a_block_reference_that_differs_from_its_output() -> None:
+ """A block that ran in two loop iterations: `<label>` merges both passes, `<label>_output` is the last."""
+ run_context = _run_context()
+ await run_context.register_output_parameter_value_post_execution(
+ _output_parameter("step_output"), {"first_pass": "a"}
+ )
+ await run_context.register_output_parameter_value_post_execution(
+ _output_parameter("step_output"), {"second_pass": "b"}
+ )
+
+ snapshot = BranchEvaluationContext(
+ workflow_run_context=run_context, block_label="cond"
+ ).build_llm_safe_context_snapshot()
+
+ assert snapshot["step"] == {"first_pass": "a", "second_pass": "b"}
+ assert snapshot["step_output"] == {"second_pass": "b"}
+
+
@pytest.mark.asyncio
async def test_jinja_render_failure_falls_back_to_extraction_block() -> None:
block = _conditional_block()
diff --git a/tests/unit/workflow/test_execute_single_block_phases.py b/tests/unit/workflow/test_execute_single_block_phases.py
index c1aed16c0..4c81b937f 100644
--- a/tests/unit/workflow/test_execute_single_block_phases.py
+++ b/tests/unit/workflow/test_execute_single_block_phases.py
@@ -14,20 +14,23 @@ from contextlib import asynccontextmanager
from datetime import UTC, datetime
from types import SimpleNamespace
from typing import Any
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from skyvern.forge import app
from skyvern.forge.sdk.db.enums import BrowserSeedSource
+from skyvern.forge.sdk.workflow.context_manager import BlockOutcome, WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import (
Block,
BranchCondition,
CodeBlock,
ConditionalBlock,
+ ForLoopBlock,
JinjaBranchCriteria,
LoginBlock,
NavigationBlock,
+ WaitBlock,
)
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
@@ -42,6 +45,7 @@ from skyvern.services import script_service
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action
from skyvern.webeye.browser_artifacts import BrowserArtifacts
+from tests.unit.fake_workflow_run_context import FakeWorkflowRunContext
def _output_parameter(key: str) -> OutputParameter:
@@ -370,7 +374,7 @@ def _wire_real_execute_safe(
monkeypatch: pytest.MonkeyPatch,
status_holder: list[WorkflowRunStatus],
*,
- execute: AsyncMock,
+ execute: AsyncMock | None = None,
pause_point: str | None = None,
) -> _PausableAwait:
pause = _PausableAwait()
@@ -397,7 +401,8 @@ def _wire_real_execute_safe(
pause if pause_point == "artifact" else AsyncMock(),
)
monkeypatch.setattr(Block, "_generate_workflow_run_block_description", AsyncMock())
- monkeypatch.setattr(NavigationBlock, "execute", execute)
+ if execute is not None:
+ monkeypatch.setattr(NavigationBlock, "execute", execute)
return pause
@@ -1013,6 +1018,70 @@ async def test_adaptive_caching_conditional_records_conditional_episode(monkeypa
assert should_stop is False
+ ("first_expression", "plan", "branch_taken", "records_episode"),
+ [
+ pytest.param("{{ plan == 'pro' }}", "pro", "paid", True, id="branch_matched"),
+ pytest.param("{{ plan == 'pro' }}", "basic", "free", True, id="every_condition_false_takes_default"),
+ pytest.param("{{ plan == 'pro' || plan == 'team' }}", "basic", "free", False, id="errored_takes_default"),
+ pytest.param("{{ plan == 'pro' || plan == 'team' }}", "trial", "trial", False, id="errored_then_later_match"),
+ ],
+)
+async def test_conditional_episode_is_recorded_only_when_every_walked_branch_was_evaluated(
+ monkeypatch: pytest.MonkeyPatch,
+ first_expression: str,
+ plan: str,
+ branch_taken: str,
+ records_episode: bool,
+) -> None:
+ block = ConditionalBlock(
+ label="cond",
+ output_parameter=_output_parameter("cond_output"),
+ branch_conditions=[
+ BranchCondition(criteria=JinjaBranchCriteria(expression=first_expression), next_block_label="paid"),
+ BranchCondition(criteria=JinjaBranchCriteria(expression="{{ plan == 'trial' }}"), next_block_label="trial"),
+ BranchCondition(is_default=True, next_block_label="free"),
+ ],
+ )
+ # The block itself runs, so the episode gate is exercised against the output a real evaluation writes.
+ _wire_real_execute_safe(monkeypatch, [WorkflowRunStatus.running])
+ monkeypatch.setattr(
+ app.WORKFLOW_CONTEXT_MANAGER,
+ "get_workflow_run_context",
+ MagicMock(return_value=FakeWorkflowRunContext(values={"plan": plan})),
+ )
+ monkeypatch.setattr(ConditionalBlock, "record_output_parameter_value", AsyncMock())
+ monkeypatch.setattr(WorkflowService, "_mark_script_fallback_triggered", AsyncMock())
+ create_episode = AsyncMock(return_value=SimpleNamespace(episode_id="cep_1"))
+ update_episode = AsyncMock()
+ monkeypatch.setattr(app.DATABASE.scripts, "create_fallback_episode", create_episode)
+ monkeypatch.setattr(app.DATABASE.scripts, "update_fallback_episode", update_episode)
+
+ _, _, block_result, should_stop, branch_metadata = await _run_single_block(
+ WorkflowService(),
+ block,
+ workflow=_adaptive_workflow(),
+ is_script_run=True,
+ script_blocks_by_label={"cond": _script_block("cond", "True", requires_agent=True)},
+ )
+
+ # Every case routes and reports completed, including the two whose first branch could not be evaluated.
+ assert block_result is not None
+ assert block_result.status == BlockStatus.completed
+ assert branch_metadata is not None
+ assert branch_metadata["branch_taken"] == branch_taken
+ assert should_stop is False
+ if records_episode:
+ create_episode.assert_awaited_once()
+ assert create_episode.await_args.kwargs["fallback_type"] == "conditional_agent"
+ assert create_episode.await_args.kwargs["agent_actions"]["branch_taken"] == branch_taken
+ update_episode.assert_awaited_once_with(episode_id="cep_1", organization_id="org_test", fallback_succeeded=True)
+ else:
+ create_episode.assert_not_awaited()
+ update_episode.assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_enrich_fallback_episode_excludes_decision_row_from_agent_action_count(
monkeypatch: pytest.MonkeyPatch,
@@ -1060,3 +1129,161 @@ async def test_enrich_fallback_episode_excludes_decision_row_from_agent_action_c
update_episode.await_args.kwargs["agent_actions"]["failure_reason"]
== script_service.VERIFIER_SWAP_FAILURE_REASON
)
+
+
+def _real_run_context(monkeypatch: pytest.MonkeyPatch) -> WorkflowRunContext:
+ context = WorkflowRunContext(
+ workflow_title="test",
+ workflow_id="wf",
+ workflow_permanent_id="wpid_test",
+ workflow_run_id="wr_test",
+ aws_client=AsyncMock(),
+ )
+ monkeypatch.setattr(app.WORKFLOW_CONTEXT_MANAGER, "get_workflow_run_context", MagicMock(return_value=context))
+ return context
+
+
+async def test_outcome_record_failure_does_not_fail_completed_block(monkeypatch: pytest.MonkeyPatch) -> None:
+ context = _real_run_context(monkeypatch)
+ monkeypatch.setattr(context, "mask_secrets_in_data", MagicMock(side_effect=RuntimeError("Mask unavailable")))
+ block = _navigation_block("nav")
+ result = BlockResult(
+ success=True,
+ status=BlockStatus.completed,
+ failure_reason="diagnostic",
+ output_parameter=block.output_parameter,
+ workflow_run_block_id="wrb_nav",
+ )
+ monkeypatch.setattr(NavigationBlock, "_execute_to_block_result", AsyncMock(return_value=result))
+
+ assert await block.execute_safe("wr_test") is result
+ assert context.get_block_outcome("nav") is None
+
+
+ ("status", "error_codes", "failure_reason", "ends_run"),
+ [
+ (BlockStatus.completed, [], None, False),
+ (BlockStatus.failed, ["AUTH_FAILURE"], "login rejected", True),
+ (BlockStatus.failed, [], "login rejected", True),
+ (BlockStatus.terminated, [], "nothing left to do", True),
+ (BlockStatus.canceled, [], None, True),
+ (BlockStatus.timed_out, [], "page never loaded", True),
+ ],
+)
+async def test_engine_records_the_same_outcome_shape_for_every_terminal_status(
+ monkeypatch: pytest.MonkeyPatch,
+ status: BlockStatus,
+ error_codes: list[str],
+ failure_reason: str | None,
+ ends_run: bool,
+) -> None:
+ block = _navigation_block("nav")
+ result = BlockResult(
+ success=status is BlockStatus.completed,
+ output_parameter=block.output_parameter,
+ status=status,
+ failure_reason=failure_reason,
+ error_codes=error_codes,
+ workflow_run_block_id="wrb_nav",
+ )
+ _wire_real_execute_safe(monkeypatch, [WorkflowRunStatus.running], execute=AsyncMock(return_value=result))
+ context = _real_run_context(monkeypatch)
+ service = WorkflowService()
+ for finalizer in (
+ "mark_workflow_run_as_failed_if_not_final",
+ "mark_workflow_run_as_terminated_if_not_final",
+ "mark_workflow_run_as_canceled",
+ ):
+ monkeypatch.setattr(service, finalizer, AsyncMock(return_value=_workflow_run()))
+
+ _, _, block_result, should_stop, _ = await _run_single_block(service, block)
+
+ assert block_result is result
+ assert should_stop is ends_run
+ assert context.get_block_outcome("nav") == BlockOutcome(
+ status=status, error_codes=error_codes, failure_reason=failure_reason
+ )
+ assert context.get_block_outcome("never_ran") is None
+
+
+async def test_a_block_the_run_ended_before_is_recorded_as_skipped(monkeypatch: pytest.MonkeyPatch) -> None:
+ block = _navigation_block("nav")
+ execute = AsyncMock()
+ status_holder = [WorkflowRunStatus.running]
+ pause = _wire_real_execute_safe(monkeypatch, status_holder, execute=execute, pause_point="screenshot")
+ context = _real_run_context(monkeypatch)
+
+ run_task = asyncio.create_task(_run_single_block(WorkflowService(), block))
+ await pause.entered.wait()
+ status_holder[0] = WorkflowRunStatus.completed
+ pause.release.set()
+ _, _, block_result, _, _ = await run_task
+
+ execute.assert_not_awaited()
+ assert block_result is not None and block_result.status is BlockStatus.skipped
+ assert context.get_block_outcome("nav") == BlockOutcome(
+ status=BlockStatus.skipped, error_codes=[], failure_reason=None
+ )
+
+
+async def test_loop_child_outcome_is_its_last_iteration_and_the_loop_records_its_own(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ child = WaitBlock(
+ label="child", output_parameter=_output_parameter("child_output"), wait_sec=1, continue_on_failure=True
+ )
+ loop = ForLoopBlock(label="each_row", output_parameter=_output_parameter("each_row_output"), loop_blocks=[child])
+ monkeypatch.setattr(
+ app.DATABASE.observer,
+ "create_workflow_run_block",
+ AsyncMock(return_value=SimpleNamespace(workflow_run_block_id="wrb_1")),
+ )
+ monkeypatch.setattr(app.BROWSER_MANAGER, "get_for_workflow_run", MagicMock(return_value=None))
+ monkeypatch.setattr(Block, "_generate_workflow_run_block_description", AsyncMock())
+ monkeypatch.setattr(ForLoopBlock, "get_loop_over_parameter_values", AsyncMock(return_value=["r1", "r2"]))
+ monkeypatch.setattr(ForLoopBlock, "get_loop_block_context_parameters", lambda self, *args, **kwargs: [])
+ monkeypatch.setattr(ForLoopBlock, "_snapshot_loop_baseline_pages", AsyncMock(return_value=None))
+ monkeypatch.setattr(ForLoopBlock, "_reset_browser_tabs_for_iteration", AsyncMock())
+ monkeypatch.setattr(ForLoopBlock, "_persist_partial_loop_output", AsyncMock())
+ context = _real_run_context(monkeypatch)
+ iteration_results = iter(
+ [(BlockStatus.failed, ["ROW_REJECTED"], "row rejected"), (BlockStatus.completed, [], None)]
+ )
+ seen_before_second_iteration: list[BlockOutcome | None] = []
+
+ async def run_child(
+ self: WaitBlock, workflow_run_id: str, workflow_run_block_id: str, **kwargs: Any
+ ) -> BlockResult:
+ status, error_codes, failure_reason = next(iteration_results)
+ if status is BlockStatus.completed:
+ seen_before_second_iteration.append(context.get_block_outcome("child"))
+ return await self.build_block_result(
+ success=status is BlockStatus.completed,
+ failure_reason=failure_reason,
+ status=status,
+ error_codes=error_codes or None,
+ workflow_run_block_id=workflow_run_block_id,
+ )
+
+ monkeypatch.setattr(WaitBlock, "execute", run_child)
+ with patch("skyvern.forge.sdk.workflow.models.block.skyvern_context") as block_context:
+ block_context.current.return_value = None
+ _, _, block_result, should_stop, _ = await _run_single_block(WorkflowService(), loop)
+
+ assert block_result is not None and block_result.status is BlockStatus.completed
+ assert should_stop is False
+ # Each iteration writes the child's record; the one that survives is the last iteration's.
+ assert seen_before_second_iteration == [
+ BlockOutcome(status=BlockStatus.failed, error_codes=["ROW_REJECTED"], failure_reason="row rejected")
+ ]
+ assert context.get_block_outcome("child") == BlockOutcome(
+ status=BlockStatus.completed, error_codes=[], failure_reason=None
+ )
+ assert context.get_block_outcome("each_row") == BlockOutcome(
+ status=BlockStatus.completed, error_codes=[], failure_reason=None
+ )