Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Test a restored Copilot proposal on its stored bytes, with no re-save needed (SKY-17668) (#8801)
Andrew Neilson778ae699bc0e
diff --git a/alembic/versions/2026_10_04_0635-2b0a571b6e5e_index_debug_sessions_lookup.py b/alembic/versions/2026_10_04_0635-2b0a571b6e5e_index_debug_sessions_lookup.pynew file mode 100644index 000000000..03270ed3c--- /dev/null+++ b/alembic/versions/2026_10_04_0635-2b0a571b6e5e_index_debug_sessions_lookup.py@@ -0,0 +1,65 @@+"""index debug_sessions lookup++Revision ID: 2b0a571b6e5e+Revises: 6fbee6664b86+Create Date: 2026-10-04T06:35:37.146186+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "2b0a571b6e5e"+down_revision: Union[str, None] = "6fbee6664b86"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None++INDEX_NAME = "ix_debug_sessions_org_wpid_user_created_at"+_TABLE = "debug_sessions"+++def _index_is_valid() -> bool | None:+ return (+ op.get_bind()+ .execute(+ sa.text("SELECT indisvalid FROM pg_catalog.pg_index WHERE indexrelid = to_regclass(:name)"),+ {"name": INDEX_NAME},+ )+ .scalar_one_or_none()+ )+++def upgrade() -> None:+ with op.get_context().autocommit_block():+ op.execute("SET statement_timeout = '10min';")+ # A failed CONCURRENTLY build leaves an INVALID index of this name, which IF NOT EXISTS would keep.+ # Assumes a single migration runner: another runner's in-flight build also reads as INVALID.+ if _index_is_valid() is False:+ op.execute(f"DROP INDEX CONCURRENTLY {INDEX_NAME};")+ # Deliberately not partial on status: the lookup binds status as a parameter, so a generic plan+ # cannot prove `status = 'created'` and would skip a partial index.+ op.create_index(+ INDEX_NAME,+ _TABLE,+ ["organization_id", "workflow_permanent_id", "user_id", "created_at"],+ unique=False,+ postgresql_concurrently=True,+ if_not_exists=True,+ )+ op.execute("RESET statement_timeout;")+++def downgrade() -> None:+ with op.get_context().autocommit_block():+ op.execute("SET statement_timeout = '10min';")+ op.drop_index(+ INDEX_NAME,+ table_name=_TABLE,+ postgresql_concurrently=True,+ if_exists=True,+ )+ op.execute("RESET statement_timeout;")diff --git a/alembic/versions/2026_10_04_0635-798a79449b1e_add_workflow_definition_sha256_to_workflow_runs.py b/alembic/versions/2026_10_04_0635-798a79449b1e_add_workflow_definition_sha256_to_workflow_runs.pynew file mode 100644index 000000000..d3a949434--- /dev/null+++ b/alembic/versions/2026_10_04_0635-798a79449b1e_add_workflow_definition_sha256_to_workflow_runs.py@@ -0,0 +1,29 @@+"""add workflow_definition_sha256 to workflow_runs++Revision ID: 798a79449b1e+Revises: 2b0a571b6e5e+Create Date: 2026-10-04T06:35:37.147784+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "798a79449b1e"+down_revision: Union[str, None] = "2b0a571b6e5e"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ op.execute("SET LOCAL lock_timeout = '5s'")+ op.add_column("workflow_runs", sa.Column("workflow_definition_sha256", sa.String(), nullable=True))+++def downgrade() -> None:+ op.execute("SET LOCAL lock_timeout = '5s'")+ op.drop_column("workflow_runs", "workflow_definition_sha256")diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsxindex 7a2d22a3f..49744152f 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx@@ -34,6 +34,7 @@ type StreamBody = {cancel_token?: string;supports_credential_pause?: boolean;supports_credential_pause_recovery?: boolean;+ supports_credential_generation?: boolean;credential_recovery_token?: string;};type StreamCall = {@@ -1862,6 +1863,237 @@ describe("WorkflowCopilotChat — credential card wiring", () => {expect(screen.queryByText(/Credential added/)).toBeNull();});+ it.each([+ ["unknown", /The vault didn't confirm the save/],+ ["created_not_connected", /Saved as Portal test account, not connected/],+ ] as const)(+ "registration Generate and save POSTs only the pause ids and a %s save removes the button",+ async (generateResult, message) => {+ sansApiPost.mockImplementation((path: string) =>+ Promise.resolve(+ path === "/workflow/copilot/credential-generate"+ ? { data: { result: generateResult } }+ : {},+ ),+ );+ await renderChat();+ await submit("create a test account");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));+ expect(streamCalls[0]!.body.supports_credential_generation).toBe(true);+ await act(async () => {+ streamCalls[0]!.onMessage(turnStart());+ streamCalls[0]!.onMessage(+ credentialFrame({+ reason: "credential_registration",+ login_page_urls: ["https://portal.example.com/signup"],+ registration: {+ username: "[email protected]",+ credential_name: "Portal test account",+ },+ }),+ );+ });+ await act(async () => {+ fireEvent.click(+ screen.getByRole("button", { name: "Generate and save" }),+ );+ });+ const generatePosts = () =>+ sansApiPost.mock.calls.filter(+ (call) => call[0] === "/workflow/copilot/credential-generate",+ );+ await waitFor(() => expect(generatePosts()).toHaveLength(1));+ expect(generatePosts()[0]![1]).toEqual({+ turn_id: "turn-1",+ workflow_copilot_chat_id: "chat-1",+ resume_token: "rt-abc",+ });+ expect(await screen.findAllByText(message)).not.toHaveLength(0);+ expect(+ screen.queryByRole("button", { name: "Generate and save" }),+ ).toBeNull();+ expect(credentialResponsePosts()).toHaveLength(0);+ expect(+ screen.queryByRole("button", { name: "Skip for now" }) === null,+ ).toBe(generateResult === "created_not_connected");+ },+ );++ const raiseRegistrationCard = async () => {+ await renderChat();+ await submit("create a test account");+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));+ await act(async () => {+ streamCalls[0]!.onMessage(turnStart());+ streamCalls[0]!.onMessage(+ credentialFrame({+ reason: "credential_registration",+ login_page_urls: ["https://portal.example.com/signup"],+ registration: {+ username: "[email protected]",+ credential_name: "Portal test account",+ },+ }),+ );+ });+ await waitFor(() => expect(credentialsGets().length).toBeGreaterThan(0));+ };++ it("registration unknown reloads the saved logins so a save that landed late can be picked", async () => {+ const extendedDeadline = new Date(Date.now() + 600_000).toISOString();+ sansApiPost.mockImplementation((path: string) =>+ Promise.resolve(+ path === "/workflow/copilot/credential-generate"+ ? {+ data: {+ result: "unknown",+ expires_at: extendedDeadline,+ },+ }+ : {},+ ),+ );+ await raiseRegistrationCard();+ const listsBefore = credentialsGets().length;+ credentialsData.current = [+ {+ credential_id: "cred-new",+ name: "Portal test account",+ tested_url: null,+ },+ ];+ await act(async () => {+ fireEvent.click(+ screen.getByRole("button", { name: "Generate and save" }),+ );+ });++ await waitFor(() =>+ expect(credentialsGets().length).toBeGreaterThan(listsBefore),+ );+ expect(+ (+ await screen.findAllByText(+ /didn't confirm the save. Check your credentials/,+ )+ ).length,+ ).toBeGreaterThan(0);+ expect((await screen.findAllByText("Saved logins")).length).toBeGreaterThan(+ 0,+ );+ expect(credentialResponsePosts()).toHaveLength(0);+ });++ it("registration Generate whose POST fails reads the card back instead of offering a retry", async () => {+ sansApiPost.mockImplementation((path: string) =>+ path === "/workflow/copilot/credential-generate"+ ? Promise.reject(new Error("gateway timeout"))+ : Promise.resolve({}),+ );+ await raiseRegistrationCard();+ historyResponse.data = {+ ...historyResponse.data,+ pending_credential_requests: [+ credentialFrame({+ reason: "credential_registration",+ login_page_urls: ["https://portal.example.com/signup"],+ registration: {+ username: "[email protected]",+ credential_name: "Portal test account",+ attempted: true,+ outcome: "unknown",+ },+ }),+ ],+ };+ await act(async () => {+ fireEvent.click(+ screen.getByRole("button", { name: "Generate and save" }),+ );+ });++ expect(+ (await screen.findAllByText(/The vault didn't confirm the save/)).length,+ ).toBeGreaterThan(0);+ expect(+ apiGet.mock.calls.some(+ ([path]) => path === "/workflow/copilot/chat-history",+ ),+ ).toBe(true);+ expect(+ screen.queryByRole("button", { name: "Generate and save" }),+ ).toBeNull();+ expect(toastFn).not.toHaveBeenCalled();+ });++ it("registration Generate whose POST fails keeps reading the card until the save settles", async () => {+ sansApiPost.mockImplementation((path: string) =>+ path === "/workflow/copilot/credential-generate"+ ? Promise.reject(new Error("gateway timeout"))+ : Promise.resolve({}),+ );+ await raiseRegistrationCard();+ const registrationCard = (+ registration: Partial<+ NonNullable<WorkflowCopilotCredentialRequiredUpdate["registration"]>+ >,+ expiresAt?: string,+ ) =>+ credentialFrame({+ reason: "credential_registration",+ login_page_urls: ["https://portal.example.com/signup"],+ registration: {+ username: "[email protected]",+ credential_name: "Portal test account",+ attempted: true,+ ...registration,+ },+ ...(expiresAt ? { expires_at: expiresAt } : {}),+ });+ historyResponse.data = {+ ...historyResponse.data,+ pending_credential_requests: [registrationCard({})],+ };+ const historyReads = () =>+ apiGet.mock.calls.filter(+ ([path]) => path === "/workflow/copilot/chat-history",+ ).length;+ vi.useFakeTimers();+ const readsBefore = historyReads();+ await act(async () => {+ fireEvent.click(+ screen.getByRole("button", { name: "Generate and save" }),+ );+ await vi.advanceTimersByTimeAsync(0);+ });++ expect(historyReads()).toBeGreaterThan(readsBefore);+ expect(screen.queryByText(/The vault didn't confirm the save/)).toBeNull();+ expect(+ screen.queryByRole("button", { name: "Generate and save" }),+ ).toBeNull();++ const extendedDeadline = new Date(Date.now() + 600_000).toISOString();+ historyResponse.data = {+ ...historyResponse.data,+ pending_credential_requests: [+ registrationCard({ outcome: "unknown" }, extendedDeadline),+ ],+ };+ await act(async () => vi.advanceTimersByTimeAsync(2_000));++ expect(+ screen.getAllByText(/The vault didn't confirm the save/).length,+ ).toBeGreaterThan(0);+ await act(async () => vi.advanceTimersByTimeAsync(300_001));+ expect(+ screen.getAllByText(/The vault didn't confirm the save/).length,+ ).toBeGreaterThan(0);+ expect(screen.queryByText("Timed out")).toBeNull();+ expect(credentialResponsePosts()).toHaveLength(0);+ expect(toastFn).not.toHaveBeenCalled();+ });+it("connect with an existing matched credential POSTs the credential_id", async () => {credentialsData.current = [{diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsxindex c1e848b73..8625f005c 100644--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx@@ -123,6 +123,7 @@ import {WorkflowCopilotWorkflowDraftUpdate,WorkflowCopilotCodegenProgressUpdate,WorkflowCopilotCredentialRequiredUpdate,+ WorkflowCopilotCredentialGenerateResult,WorkflowCopilotCredentialResponseResult,WorkflowCopilotCredentialPauseResolvedUpdate,WorkflowCopilotTitleUpdate,@@ -378,6 +379,9 @@ const RECOVERY_POLL_STEADY_MS = 30_000;// up the real reply if the turn finishes after all.const RECOVERY_POLL_BUDGET_MS = 1_500_000;const CANONICAL_READ_TIMEOUT_MS = 5_000;+// Matches the server's MANUAL_SIGN_IN_CLAIM_SECONDS, which bounds a Generate and save.+const REGISTRATION_SAVE_WINDOW_MS = 120_000;+const REGISTRATION_SAVE_POLL_MS = 2_000;const INTERRUPTED_TERMINAL_REASON = "interrupted";// Only an older backend writes an interrupted row a late reply may still replace.const isReplaceableInterruptedRow = (@@ -2548,10 +2552,79 @@ export function WorkflowCopilotChat({const recoverCredentialTurn = useRef<(chatId: string, turnId: string) => boolean>(() => false);+ const updateRegistration = useCallback(+ (+ frame: WorkflowCopilotCredentialRequiredUpdate,+ update: Partial<+ NonNullable<WorkflowCopilotCredentialRequiredUpdate["registration"]>+ >,+ expiresAt?: string | null,+ ) => {+ // The card's Generate button, its countdown, and a reloaded card all read the frame.+ const apply = (candidate: WorkflowCopilotCredentialRequiredUpdate) =>+ candidate.resume_token === frame.resume_token && candidate.registration+ ? {+ ...candidate,+ registration: { ...candidate.registration, ...update },+ expires_at: expiresAt ?? candidate.expires_at,+ }+ : candidate;+ setLivePauseFrame((prev) => (prev ? apply(prev) : prev));+ setRecoveredPauseFrames((prev) => prev.map(apply));+ },+ [],+ );+ // A Generate whose answer never arrived may still be saving: read the card back until the save settles.+ const refreshRegistrationCard = useCallback(+ async (frame: WorkflowCopilotCredentialRequiredUpdate) => {+ const stopAt = Date.now() + REGISTRATION_SAVE_WINDOW_MS;+ const generation = recoveryGeneration.current;+ const stale = () =>+ recoveryGeneration.current !== generation ||+ (workflowCopilotChatIdRef.current !== null &&+ workflowCopilotChatIdRef.current !== frame.workflow_copilot_chat_id);+ setCredentialsReloadKey((key) => key + 1);+ try {+ const client = await getClient(credentialGetter, "sans-api-v1");+ for (;;) {+ if (stale()) return;+ const response =+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(+ client,+ workflowPermanentId,+ {+ params: {+ workflow_copilot_chat_id: frame.workflow_copilot_chat_id,+ },+ },+ );+ const current = response.data.pending_credential_requests?.find(+ (candidate) => candidate.resume_token === frame.resume_token,+ );+ if (stale() || !current?.registration) return;+ updateRegistration(frame, current.registration, current.expires_at);+ if (current.registration.outcome) {+ setCredentialsReloadKey((key) => key + 1);+ return;+ }+ if (Date.now() >= stopAt) return;+ await new Promise((resolve) =>+ setTimeout(resolve, REGISTRATION_SAVE_POLL_MS),+ );+ }+ } catch (error) {+ console.error(+ "Failed to refresh the credential card:",+ error instanceof Error ? error.message : String(error),+ );+ }+ },+ [credentialGetter, updateRegistration, workflowPermanentId],+ );const respondToCredentialPause = useCallback(async (frame: WorkflowCopilotCredentialRequiredUpdate,- action: "connected" | "skip" | "signing_in" | "signed_in",+ action: "connected" | "skip" | "signing_in" | "signed_in" | "generate",credentialId?: string,name?: string,): Promise<WorkflowCopilotCredentialResponseResult | undefined> => {@@ -2573,22 +2646,40 @@ export function WorkflowCopilotChat({// Copilot routes live on base_router (no /api/v1 prefix), like cancel.const client = await getClient(credentialGetter, "sans-api-v1");if (!isCopilotTurnCurrent(reservation)) return;- const response =- await client.post<WorkflowCopilotCredentialResponseResult>(- "/workflow/copilot/credential-response",- {- turn_id: frame.turn_id,- workflow_copilot_chat_id: frame.workflow_copilot_chat_id,- resume_token: frame.resume_token,- action,- credential_id: action === "connected" ? credentialId : undefined,- },- );- // A backend that predates sign-in answers 204 with no body.- const result: WorkflowCopilotCredentialResponseResult = response.data- ?.result- ? response.data- : { result: "accepted" };+ const pauseIds = {+ turn_id: frame.turn_id,+ workflow_copilot_chat_id: frame.workflow_copilot_chat_id,+ resume_token: frame.resume_token,+ };+ let generated: WorkflowCopilotCredentialGenerateResult | null = null;+ let result: WorkflowCopilotCredentialResponseResult = {+ result: "accepted",+ };+ if (action === "generate") {+ // The server allows one attempt per card, so Generate goes away as soon as it is sent.+ updateRegistration(frame, { attempted: true });+ generated = (+ await client.post<WorkflowCopilotCredentialGenerateResult>(+ "/workflow/copilot/credential-generate",+ pauseIds,+ )+ ).data;+ // Any answer, even an unconfirmed one, can mean a new saved login the picker should list.+ setCredentialsReloadKey((key) => key + 1);+ } else {+ const response =+ await client.post<WorkflowCopilotCredentialResponseResult>(+ "/workflow/copilot/credential-response",+ {+ ...pauseIds,+ action,+ credential_id:+ action === "connected" ? credentialId : undefined,+ },+ );+ // A backend that predates sign-in answers 204 with no body.+ if (response.data?.result) result = response.data;+ }if (!isCopilotTurnCurrent(reservation) ||recoveryGeneration.current !== generation ||@@ -2604,6 +2695,25 @@ export function WorkflowCopilotChat({frame.turn_id,);}+ if (generated && generated.result !== "connected") {+ updateRegistration(+ frame,+ { outcome: generated.result },+ generated.expires_at,+ );+ // The pause ended before the save landed, so its resume token answers nothing now.+ if (generated.result === "created_not_connected") {+ setPauseCardResolutions((prev) =>+ prev[frame.resume_token]+ ? prev+ : withCappedResolution(prev, frame.resume_token, {+ outcome: "timeout",+ name: generated.name ?? undefined,+ }),+ );+ }+ return result;+ }if (action === "signing_in" ||result.result === "no_sign_in_found" ||@@ -2611,8 +2721,13 @@ export function WorkflowCopilotChat({) {return result;}- const resolution: CredentialResolution =- action === "connected"+ const resolution: CredentialResolution = generated+ ? {+ outcome: "connected",+ credentialId: generated.credential_id ?? undefined,+ name: generated.name ?? undefined,+ }+ : action === "connected"? { outcome: "connected", credentialId, name }: action === "signed_in"? { outcome: "signed_in" }@@ -2643,6 +2758,11 @@ export function WorkflowCopilotChat({error,frame.workflow_copilot_chat_id,);+ // Generate is never retried: the save may have gone through, so read the card back instead.+ if (action === "generate") {+ void refreshRegistrationCard(frame);+ return;+ }toast({title: "Couldn't send your credential response",description: "Please try again.",@@ -2652,7 +2772,13 @@ export function WorkflowCopilotChat({credentialResponseInFlight.current = false;}},- [credentialGetter, isCopilotTurnCurrent, logCopilotRequestFailure],+ [+ credentialGetter,+ isCopilotTurnCurrent,+ logCopilotRequestFailure,+ refreshRegistrationCard,+ updateRegistration,+ ],);const startManualSignIn = useCallback(async (frame: WorkflowCopilotCredentialRequiredUpdate) => {@@ -8416,6 +8542,7 @@ export function WorkflowCopilotChat({selected_block_label: readSelectedBlockLabel(),keep_pending_proposal: Boolean(pendingProposalTurnId),supports_credential_pause: true,+ supports_credential_generation: true,supports_credential_pause_recovery: Boolean(credentialRecoveryToken,),@@ -9462,6 +9589,7 @@ export function WorkflowCopilotChat({: openCredentialModal(frame, frame.turn_id)}onSkip={() => void respondToCredentialPause(frame, "skip")}+ onGenerate={() => void respondToCredentialPause(frame, "generate")}signIn={frame.sign_in_browser_session_id &&frame.sign_in_browser_session_id === liveBrowserSessionIddiff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.fixtures.ts b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.fixtures.tsindex d34d72432..283ce728b 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.fixtures.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.fixtures.ts@@ -60,6 +60,15 @@ export const CREDENTIAL_REQUIRED_FRAME_BY_REASON = {message: "The site rejected the saved one-time code.",credential_refs: ["cred_hn"],}),+ credential_registration: buildCredentialRequiredFrame({+ reason: "credential_registration",+ message: "Create the test account you asked for.",+ login_page_urls: ["https://portal.example.com/signup"],+ registration: {+ username: "[email protected]",+ credential_name: "Portal test account",+ },+ }),} as const;// No dynamic ask text; only a richer, timed pause signal supplies one.diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsxindex 802c1b3a0..b826d4131 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx@@ -429,6 +429,69 @@ describe("CredentialCard sign in myself", () => {});});+describe("CredentialCard registration", () => {+ const registrationFrame =+ CREDENTIAL_REQUIRED_FRAME_BY_REASON.credential_registration;++ it("shows the exact sign-up page, username and saved name, and Generate and save answers it", () => {+ const onGenerate = vi.fn();+ const onSkip = vi.fn();+ render(+ <CredentialCard+ frame={registrationFrame}+ mode="inline-pause"+ onConnect={vi.fn()}+ onSkip={onSkip}+ onGenerate={onGenerate}+ />,+ );+ expect(+ screen.getByText("Create a login for https://portal.example.com"),+ ).toBeTruthy();+ expect(+ screen.getByText("Sign-up page: https://portal.example.com/signup"),+ ).toBeTruthy();+ expect(screen.getByText("Username: [email protected]")).toBeTruthy();+ expect(screen.getByText("Saved as: Portal test account")).toBeTruthy();+ fireEvent.click(screen.getByRole("button", { name: "Generate and save" }));+ expect(onGenerate).toHaveBeenCalledTimes(1);+ expect(onSkip).not.toHaveBeenCalled();+ });++ it.each([+ ["rejected", /Nothing was saved/],+ ["unknown", /The vault didn't confirm the save/],+ ["created_not_connected", /Saved as Portal test account, not connected/],+ ] as const)(+ "after a %s attempt, hides Generate and says what happened",+ (outcome, message) => {+ render(+ <CredentialCard+ frame={{+ ...registrationFrame,+ registration: {+ ...registrationFrame.registration!,+ attempted: true,+ outcome,+ },+ }}+ mode="inline-pause"+ onConnect={vi.fn()}+ onSkip={vi.fn()}+ onGenerate={vi.fn()}+ />,+ );+ expect(+ screen.queryByRole("button", { name: "Generate and save" }),+ ).toBeNull();+ expect(screen.getAllByText(message)).not.toHaveLength(0);+ expect(+ screen.getByRole("button", { name: "Connect credential" }),+ ).toBeTruthy();+ },+ );+});+describe("CredentialCard terminal org-credential picker", () => {it("fetches the org credentials and renders them as a picker, in API order", async () => {credsData.current = [diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsxindex be8e194c4..bc87576ac 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx@@ -61,7 +61,16 @@ export type CredentialRequiredReason =| "assistant_directed"| "missing_credential_run_failure"| "credential_missing_totp"- | "credential_rejected_by_site";+ | "credential_rejected_by_site"+ | "credential_registration";++// What Generate and save would store. Never carries a password; the server generates it.+export interface CredentialRegistrationDetails {+ username: string;+ credential_name: string;+ attempted?: boolean;+ outcome?: "rejected" | "unknown" | "created_not_connected" | null;+}export interface CredentialRequiredFrame {type: "credential_required";@@ -75,6 +84,7 @@ export interface CredentialRequiredFrame {timeout_seconds?: number;expires_at?: string;signing_in?: boolean;+ registration?: CredentialRegistrationDetails | null;timestamp?: string;}@@ -135,6 +145,8 @@ export interface CredentialCardProps {tray?: AttentionTrayPresentation;// Offered only when the card's browser is the one on screen, so the user signs in where the agent looks.signIn?: ManualSignInOffer;+ // Registration cards only: saves a server-generated password under the frame's username and name.+ onGenerate?: () => void;}export interface ManualSignInOffer {@@ -173,6 +185,8 @@ export const CREDENTIAL_WHY_LINE_BY_REASON: Record<"This saved login has no 2FA method, so the workflow can't pass the verification step. Add one in the credential editor — codes never go through chat.",credential_rejected_by_site:"Update the saved password or one-time code here and I'll try the sign-in again.",+ credential_registration:+ "Generate and save creates a strong password and stores it in your credentials — it never appears in chat. Saving it does not create the account; I'll still submit the sign-up form.",};// Mirrors the credentials route: it caps `search` at 200 characters and pages at 100. A longer term@@ -774,6 +788,7 @@ function CredentialAskCard({canChange = false,tray,signIn,+ onGenerate,}: Readonly<CredentialCardProps>) {// Terminal mode never expires by design: its signal carries no timeout/expiry// semantics at all, so there is nothing to compare "now" against. Only a@@ -925,7 +940,15 @@ function CredentialAskCard({/>);case "timeout":- return <ResolvedCredentialCard tone="warn" {...TIMEOUT_OUTCOME} />;+ return frame.registration?.outcome === "created_not_connected" ? (+ <ResolvedCredentialCard+ tone="warn"+ title={`Saved as ${frame.registration.credential_name}, not connected`}+ detail="The request ended before this login was connected. It is on the Credentials page."+ />+ ) : (+ <ResolvedCredentialCard tone="warn" {...TIMEOUT_OUTCOME} />+ );case "signed_in":return (<ResolvedCredentialCard@@ -1049,11 +1072,41 @@ function CredentialAskCard({}const site = siteFromLoginPageUrls(frame.login_page_urls);+ const registration = frame.registration ?? null;+ const offerGenerate = Boolean(+ registration && onGenerate && !registration.attempted,+ );+ const registrationLines = registration+ ? [+ <span key="destination" className="text-[11px] leading-relaxed">+ Sign-up page: {frame.login_page_urls?.[0] ?? site}+ </span>,+ <span key="username" className="text-[11px] leading-relaxed">+ Username: {registration.username}+ </span>,+ <span key="saved-as" className="text-[11px] leading-relaxed">+ Saved as: {registration.credential_name}+ </span>,+ ...(registration.outcome+ ? [+ <span key="outcome" className="text-[11px] font-medium">+ {registration.outcome === "rejected"+ ? "Nothing was saved — the credential couldn't be created. Pick or add a login instead."+ : registration.outcome === "created_not_connected"+ ? `Saved as ${registration.credential_name}, not connected.`+ : "The vault didn't confirm the save. Check your credentials before adding another."}+ </span>,+ ]+ : []),+ ]+ : [];const connectButton = (<Buttontype="button"size="sm"- variant={signIn && frame.signing_in ? "outline" : "default"}+ variant={+ (signIn && frame.signing_in) || offerGenerate ? "outline" : "default"+ }disabled={disabled}onClick={() => onConnect(undefined)}data-tour="credential-connect"@@ -1133,7 +1186,11 @@ function CredentialAskCard({tray={tray}rootRef={rootRef}message={frame.message}- title={`Copilot needs to sign in to ${site}`}+ title={+ registration+ ? `Create a login for ${site}`+ : `Copilot needs to sign in to ${site}`+ }countdown={countdownActive ? (<PauseCountdown remainingMs={remainingMs} expired={expired} />@@ -1146,6 +1203,7 @@ function CredentialAskCard({>{CREDENTIAL_WHY_LINE_BY_REASON[frame.reason] ?? SIGN_IN_WHY_LINE}</span>,+ ...registrationLines,...(mode === "terminal"? [<span@@ -1159,6 +1217,16 @@ function CredentialAskCard({]}footer={<>+ {offerGenerate ? (+ <Button+ type="button"+ size="sm"+ disabled={disabled}+ onClick={() => onGenerate?.()}+ >+ Generate and save+ </Button>+ ) : null}{connectButton}{signIn ? (<Buttondiff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.tsindex 96df2f1a5..16c34f620 100644--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts@@ -168,6 +168,7 @@ export interface WorkflowCopilotChatRequest {// Opt-in: only clients that can render the credential_required frame set// this, so the backend never pauses a turn a client would silently drop.supports_credential_pause?: boolean;+ supports_credential_generation?: boolean;credential_recovery_token?: string;supports_credential_pause_recovery?: boolean;supports_question_tool?: boolean;@@ -435,6 +436,13 @@ export interface WorkflowCopilotCredentialRequiredUpdate {// The live browser the user may sign in to themselves; absent when the card does not offer it.sign_in_browser_session_id?: string | null;signing_in?: boolean;+ // Present on a card that offers Generate and save; never carries a password.+ registration?: {+ username: string;+ credential_name: string;+ attempted?: boolean;+ outcome?: "rejected" | "unknown" | "created_not_connected" | null;+ } | null;timestamp: string;}@@ -445,6 +453,15 @@ export interface WorkflowCopilotCredentialResponseResult {browser_profile_id?: string | null;}+export interface WorkflowCopilotCredentialGenerateResult {+ result: "connected" | "rejected" | "unknown" | "created_not_connected";+ credential_id?: string | null;+ name?: string | null;+ username?: string | null;+ // The reopened card's new deadline after "rejected" or "unknown".+ expires_at?: string | null;+}+export interface WorkflowCopilotCredentialPauseResolvedUpdate {type: "credential_pause_resolved";turn_id: string;diff --git a/skyvern/forge/log_redaction.py b/skyvern/forge/log_redaction.pyindex d441e2aac..e102c0381 100644--- a/skyvern/forge/log_redaction.py+++ b/skyvern/forge/log_redaction.py@@ -42,6 +42,7 @@ SENSITIVE_HEADERS = {"x-marketplace-api-key","x-posthog-attribution","x-copilot-credential-recovery-token",+ "stripe-signature",}# Exact field names that are always redacted. Use a set for O(1) lookup@@ -79,6 +80,11 @@ SENSITIVE_FIELDS: set[str] = {"client_secret","access_token","refresh_token",+ # Masks only billing_details, payment_method_details and receipt_email, wherever they nest. Card blocks and+ # invoice or checkout customer fields are not covered; the webhook handler's identifier-only log lines protect them.+ "billing_details",+ "payment_method_details",+ "receipt_email",} | SENSITIVE_HEADERS_PROXY_OBSERVABILITY_FIELDS = {field.value: field for field in ProxyObservabilityField}diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.pyindex b0bdb67ce..630874e31 100644--- a/skyvern/forge/request_logging.py+++ b/skyvern/forge/request_logging.py@@ -57,6 +57,8 @@ _SENSITIVE_ENDPOINTS = {"POST /v1/workflow/copilot/question-response","POST /v1/workflow/copilot/credential-response","POST /v1/workflow/copilot/convert-yaml-to-blocks",+ # Payment-provider events carry customer billing details (name, email, address, card metadata).+ "POST /api/v1/stripe_webhook",}_SENSITIVE_ENDPOINT_PATTERNS = (re.compile(r"^(?:POST|PUT) /(?:api/)?v1/credentials(?:/.*)?$"),diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex c99d56342..7a9588428 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -124,6 +124,8 @@ from skyvern.forge.sdk.copilot.context import (from skyvern.forge.sdk.copilot.credential_pause import credential_recovery_token_digestfrom skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failurefrom skyvern.forge.sdk.copilot.enforcement import (+ FINAL_REPLY_OBSERVATION,+ RAW_SECRET_REPLY_WITHHELD_OBSERVATION,CopilotNonRetriableNavError,_elapsed_run_seconds,artifact_health_blocked,@@ -4359,6 +4361,7 @@ async def _run_agent_loop_with_surface(is_fallback: bool = False,session: Session | None = None,final_reply: bool = False,+ final_reply_observation: str = FINAL_REPLY_OBSERVATION,) -> Any:# No model owns the attempt until setup completes and enforcement is ready to# enter the model loop. This also clears a prior model before fallback setup.@@ -4429,6 +4432,7 @@ async def _run_agent_loop_with_surface(session=session,hooks=FinalReplyRunHooks(ctx),run_config=run_config,+ observation=final_reply_observation,)else:result = await run_with_enforcement(@@ -4577,6 +4581,44 @@ def _build_request_policy_clarification_result()+PENDING_PROPOSAL_TEST_NOT_RESTORED_REPLY = (+ "Nothing was run: the pending Copilot proposal changed before this test started, so it would not have "+ "run the proposal as offered. Reload and test the current proposal."+)+++def _build_pending_proposal_test_refusal_result(+ ctx: CopilotContext,+ prior_global_llm_context: str | None,+ prior_workflow_yaml: str | None,+) -> AgentResult:+ final_text, outcome = apply_repeated_reply_guard(+ final_text=PENDING_PROPOSAL_TEST_NOT_RESTORED_REPLY,+ attempted_kind=ResponseKind.REFUSE,+ blocked_signatures=list(ctx.blocked_reply_signatures),+ reason_code="pending_proposal_test_not_restored",+ )+ return _make_agent_result(+ ctx,+ user_response=final_text,+ updated_workflow=None,+ global_llm_context=prior_global_llm_context,+ workflow_yaml=prior_workflow_yaml or None,+ workflow_was_persisted=False,+ clear_proposed_workflow=False,+ proposal_disposition="no_proposal",+ turn_outcome=outcome,+ turn_id=ctx.turn_id,+ narrative_summary=ctx.narrative_summary,+ narrative_payload=_build_narrative_payload(+ ctx,+ terminal="response",+ terminal_message=final_text,+ narrative_summary=ctx.narrative_summary,+ ),+ )++def _agent_output_to_text(agent_output: Any) -> str:if isinstance(agent_output, str):return agent_output@@ -5274,6 +5316,7 @@ async def _run_copilot_turn_impl(selected_block_label=getattr(chat_request, "selected_block_label", None),client_supports_credential_pause=getattr(chat_request, "supports_credential_pause", False),client_supports_credential_pause_recovery=chat_request.supports_credential_pause_recovery,+ client_supports_credential_generation=chat_request.supports_credential_generation,credential_recovery_token_digest=credential_recovery_token_digest(chat_request.credential_recovery_token.get_secret_value()if chat_request.credential_recovery_token@@ -5505,6 +5548,12 @@ async def _run_copilot_turn_impl(get_skyvern_mcp_alias_map,)+ if chat_request.product_action == "test_end_to_end" and ctx.staged_workflow is None:+ # Without this gate the route-time proposal bytes would be staged, and the run would be created and then+ # refused at bind.+ LOG.info("copilot_pending_proposal_test_refused", reason="proposal_not_restored")+ return _build_pending_proposal_test_refusal_result(ctx, global_llm_context, chat_request.workflow_yaml)+validated_browser_session_id = await _resolve_live_browser_session_id(chat_request, organization_id, ctx)ctx.browser_session_id = validated_browser_session_id@@ -5759,6 +5808,7 @@ async def _run_copilot_turn_impl(*,is_fallback: bool,final_reply: bool = False,+ final_reply_observation: str = FINAL_REPLY_OBSERVATION,) -> RunResultStreaming:attempt = await _run_agent_loop_with_surface(ctx=ctx,@@ -5778,6 +5828,7 @@ async def _run_copilot_turn_impl(is_fallback=is_fallback,session=model_session,final_reply=final_reply,+ final_reply_observation=final_reply_observation,)return attempt@@ -5792,6 +5843,38 @@ async def _run_copilot_turn_impl(evaluated_reason_codes=_output_policy_reason_codes_from_guardrail_exception(exc),)+ async def _reask_after_withheld_secret_reply(exc: OutputGuardrailTripwireTriggered) -> AgentResult:+ # The SDK saves a final reply to the session only after its output guardrail passes, so the+ # withheld text is not in the history the re-ask reads. A second hit keeps the refusal.+ if _output_policy_verdict_from_guardrail_exception(exc).reason_codes != [OutputPolicyReason.RAW_SECRET_LEAK]:+ return _output_policy_blocked(exc)+ LOG.info("copilot_raw_secret_reply_reasked")+ try:+ reply = await _run_attempt(+ model_name,+ run_config,+ llm_key,+ is_fallback=False,+ final_reply=True,+ final_reply_observation=RAW_SECRET_REPLY_WITHHELD_OBSERVATION,+ )+ return await _translate_to_agent_result(+ reply,+ ctx,+ global_llm_context,+ chat_request,+ organization_id,+ final_reply=True,+ )+ except asyncio.CancelledError:+ LOG.info("Copilot run cancelled")+ return _build_cancelled_exit_result(ctx, global_llm_context)+ except OutputGuardrailTripwireTriggered as reask_exc:+ return _output_policy_blocked(reask_exc)+ except Exception as reask_error:+ LOG.warning("copilot_raw_secret_reply_reask_failed", error_type=type(reask_error).__name__)+ return _output_policy_blocked(exc)+try:from skyvern.forge.sdk.copilot.session_factory import create_copilot_session@@ -5870,7 +5953,7 @@ async def _run_copilot_turn_impl(ctx=ctx,)except OutputGuardrailTripwireTriggered as exc:- return _output_policy_blocked(exc)+ return await _reask_after_withheld_secret_reply(exc)except CopilotTurnHalt as exc:LOG.info("Copilot run stopped after typed turn halt",diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 246f748f9..0b3fc05a9 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -1247,6 +1247,7 @@ class CopilotContext(AgentContext):last_run_skipped_unbound_credentials: bool = Falseclient_supports_credential_pause: bool = Falseclient_supports_credential_pause_recovery: bool = False+ client_supports_credential_generation: bool = Falsecredential_recovery_token_digest: str | None = field(default=None, repr=False)credential_recovery_armed: bool = Falsecredential_pause_used: bool = False@@ -1259,6 +1260,9 @@ class CopilotContext(AgentContext):credential_pause_reaskable_by_run: bool = Falsecopilot_credential_pause_seconds: float = 0.0credential_pause_outcome: str | None = None+ credential_registration_outcome: Literal["rejected", "unknown"] | None = None+ # Generate and save created, or may have created, a credential; a second generate card could mint a duplicate.+ credential_generation_spent: bool = Falsecredential_pause_connected_credential_id: str | None = Nonecredential_pause_anchor_tool_call_id: str | None = None# Set while a ``request_credential`` ask is open, so tool calls issued alongside it in the same@@ -1433,6 +1437,9 @@ class CopilotContext(AgentContext):proposal_revision: int | None = Noneproposal_canonical_fingerprint: str | None = Noneproposal_workflow_run_id: str | None = None+ # A restored candidate whose request private settings differ from its own; its stored bytes cannot vouch+ # for what a test under this token would run, so binding a run to it is refused.+ settings_diverged_proposal_token: tuple[str, int] | None = None# The chat row's setting, not the turn's commit decision: the route can still refuse to apply a# staged draft at turn end. None on entrypoints that load no chat row.auto_accept: bool | None = Nonediff --git a/skyvern/forge/sdk/copilot/credential_generation.py b/skyvern/forge/sdk/copilot/credential_generation.pynew file mode 100644index 000000000..14d2a2d50--- /dev/null+++ b/skyvern/forge/sdk/copilot/credential_generation.py@@ -0,0 +1,27 @@+import secrets+import string+from typing import Literal++REGISTRATION_PASSWORD_MIN_LENGTH = 24+REGISTRATION_PASSWORD_MAX_LENGTH = 128+SAFE_SYMBOLS = "!#$%*+-=?@^_~"++RegistrationCharset = Literal["alphanumeric", "alphanumeric_symbols"]+++def _classes(charset: RegistrationCharset) -> list[str]:+ classes = [string.ascii_lowercase, string.ascii_uppercase, string.digits]+ return classes + [SAFE_SYMBOLS] if charset == "alphanumeric_symbols" else classes+++def generate_registration_password(length: int, charset: RegistrationCharset) -> str:+ # 24 characters from the 62-character alphabet already carry 142 bits; redrawing until every class+ # appears costs well under one bit.+ if not REGISTRATION_PASSWORD_MIN_LENGTH <= length <= REGISTRATION_PASSWORD_MAX_LENGTH:+ raise ValueError("password length out of range")+ classes = _classes(charset)+ alphabet = "".join(classes)+ while True:+ password = "".join(secrets.choice(alphabet) for _ in range(length))+ if all(any(char in chars for char in password) for chars in classes):+ return passworddiff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.pyindex 875ef879c..54202c91c 100644--- a/skyvern/forge/sdk/copilot/credential_pause.py+++ b/skyvern/forge/sdk/copilot/credential_pause.py@@ -48,6 +48,7 @@ from skyvern.forge.sdk.schemas.credentials import Credentialfrom skyvern.forge.sdk.schemas.workflow_copilot import (CredentialPauseResolvedOutcome,WorkflowCopilotCredentialPauseResolvedUpdate,+ WorkflowCopilotCredentialRegistration,WorkflowCopilotCredentialRequiredUpdate,WorkflowCopilotStreamMessageType,)@@ -85,7 +86,7 @@ def _credential_pause_lock_key(organization_id: str, chat_id: str, turn_id: str)# A Done claim reserves the pause while the profile is saved outside the lock; the save itself is bounded-# tighter so the claim always outlives it.+# tighter so the claim always outlives it. A Generate and save claim reuses the same window.MANUAL_SIGN_IN_CLAIM_SECONDS = 120MANUAL_SIGN_IN_SAVE_TIMEOUT_SECONDS = 90@@ -144,6 +145,8 @@ class _ActivePauseRecord:manual_sign_in: ManualSignIn | None = Noneclaim_id: str | None = Noneclaim_deadline: datetime | None = None+ registration: WorkflowCopilotCredentialRegistration | None = None+ generated_credential_id: str | None = Nonedef live_until(self) -> datetime | None:if self.status == "pending":@@ -174,6 +177,8 @@ def _encode_record(record: _ActivePauseRecord) -> str:),"claim_id": record.claim_id,"claim_deadline": record.claim_deadline.isoformat() if record.claim_deadline else None,+ "registration": record.registration.model_dump(mode="json") if record.registration else None,+ "generated_credential_id": record.generated_credential_id,})@@ -186,6 +191,7 @@ def _encode_active_pause(card: WorkflowCopilotCredentialRequiredUpdate | None = None,recovery_token_digest: str | None = None,manual_sign_in: ManualSignIn | None = None,+ registration: WorkflowCopilotCredentialRegistration | None = None,) -> str:return _encode_record(_ActivePauseRecord(@@ -195,6 +201,7 @@ def _encode_active_pause(card=card,recovery_token_digest=recovery_token_digest,manual_sign_in=manual_sign_in,+ registration=registration,))@@ -245,8 +252,17 @@ def _decode_active_pause(raw: Any) -> _ActivePauseRecord | None:card = WorkflowCopilotCredentialRequiredUpdate.model_validate(data["card"]) if data.get("card") else Noneexcept ValidationError:card = None+ try:+ registration = (+ WorkflowCopilotCredentialRegistration.model_validate(data["registration"])+ if data.get("registration")+ else None+ )+ except ValidationError:+ registration = Nonedigest = data.get("recovery_token_digest")claim_id = data.get("claim_id")+ generated_credential_id = data.get("generated_credential_id")return _ActivePauseRecord(resume_token=token,status=record_status,@@ -256,6 +272,8 @@ def _decode_active_pause(raw: Any) -> _ActivePauseRecord | None:manual_sign_in=_decode_manual_sign_in(data.get("manual_sign_in")),claim_id=claim_id if isinstance(claim_id, str) else None,claim_deadline=_decode_datetime(data.get("claim_deadline")),+ registration=registration,+ generated_credential_id=generated_credential_id if isinstance(generated_credential_id, str) else None,)@@ -355,7 +373,8 @@ async def pending_credential_requests(and card.resume_token == record.resume_tokenand card.expires_at == record.expires_at):- cards.append(card)+ registration = record.registration+ cards.append(card.model_copy(update={"registration": registration}) if registration else card)return cards@@ -431,7 +450,13 @@ async def resolve_credential_pause(async with cache.get_lock(lock_key):record = _validate_pending_pause(_decode_active_pause(await cache.get(active_key)), resume_token)ttl = _credential_pause_record_ttl(settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS)- await cache.set(active_key, _encode_active_pause(record.resume_token, record.expires_at, consumed=True), ex=ttl)+ await cache.set(+ active_key,+ _encode_active_pause(+ record.resume_token, record.expires_at, consumed=True, registration=record.registration+ ),+ ex=ttl,+ )await cache.set(credential_response_cache_key(organization_id, workflow_copilot_chat_id, turn_id),encode_credential_response(action, credential_id, record.resume_token),@@ -452,6 +477,7 @@ class CredentialPauseResolution:action: Literal["connected", "skip", "signed_in"]credential: Credential | None = Nonesigned_in: SignedInProfile | None = None+ generated: bool = Falsedef encode_credential_response(@@ -531,15 +557,20 @@ async def start_manual_sign_in()if sign_in.started:return record.expires_at- expires_at = max(- record.expires_at,- datetime.now(timezone.utc) + timedelta(seconds=settings.WORKFLOW_COPILOT_MANUAL_SIGN_IN_TIMEOUT_SECONDS),+ record = _extend_pause(+ replace(record, manual_sign_in=replace(sign_in, started=True)),+ settings.WORKFLOW_COPILOT_MANUAL_SIGN_IN_TIMEOUT_SECONDS,+ signing_in=True,)- # The recovered card must carry the record's deadline, or pending_credential_requests drops it.- card = record.card.model_copy(update={"expires_at": expires_at, "signing_in": True}) if record.card else None- record = replace(record, expires_at=expires_at, card=card, manual_sign_in=replace(sign_in, started=True))await cache.set(active_key, _encode_record(record), ex=_longest_credential_pause_record_ttl())- return expires_at+ return record.expires_at+++def _extend_pause(record: _ActivePauseRecord, seconds: int, **card_update: bool) -> _ActivePauseRecord:+ expires_at = max(record.expires_at, datetime.now(timezone.utc) + timedelta(seconds=seconds))+ # The recovered card must carry the record's deadline, or pending_credential_requests drops it.+ card = record.card.model_copy(update={"expires_at": expires_at, **card_update}) if record.card else None+ return replace(record, expires_at=expires_at, card=card)@dataclass(frozen=True)@@ -608,6 +639,87 @@ async def finish_manual_sign_in(return True+@dataclass(frozen=True)+class CredentialGenerationClaim:+ claim_id: str+ registration: WorkflowCopilotCredentialRegistration+ deadline: datetime+++async def claim_credential_generation(+ cache: BaseCache,+ *,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ resume_token: str,+) -> CredentialGenerationClaim:+ """Reserve a registration card's one Generate and save while the credential is created outside the lock."""+ active_key = credential_pause_active_key(organization_id, workflow_copilot_chat_id, turn_id)+ async with cache.get_lock(_credential_pause_lock_key(organization_id, workflow_copilot_chat_id, turn_id)):+ record = _validate_pending_pause(_decode_active_pause(await cache.get(active_key)), resume_token)+ registration = record.registration+ if registration is None:+ raise CredentialPauseRejection(+ status_code=HTTPStatus.CONFLICT,+ detail="This credential card does not offer generating a credential",+ )+ if registration.attempted:+ raise CredentialPauseRejection(+ status_code=HTTPStatus.CONFLICT,+ detail="This credential card already tried to generate a credential",+ )+ claim_id = secrets.token_hex(16)+ registration = registration.model_copy(update={"attempted": True})+ deadline = datetime.now(timezone.utc) + timedelta(seconds=MANUAL_SIGN_IN_CLAIM_SECONDS)+ claimed = replace(+ record, status="resolving", claim_id=claim_id, claim_deadline=deadline, registration=registration+ )+ await cache.set(active_key, _encode_record(claimed), ex=_longest_credential_pause_record_ttl())+ return CredentialGenerationClaim(claim_id=claim_id, registration=registration, deadline=deadline)+++async def finish_credential_generation(+ cache: BaseCache,+ *,+ organization_id: str,+ workflow_copilot_chat_id: str,+ turn_id: str,+ claim: CredentialGenerationClaim,+ credential_id: str | None,+ outcome: Literal["rejected", "unknown"] | None = None,+) -> datetime | None:+ """Connect the created credential, or reopen the card without Generate and with a fresh deadline.++ Returns the pause's deadline, or None when the claim was lost."""+ active_key = credential_pause_active_key(organization_id, workflow_copilot_chat_id, turn_id)+ if credential_id is not None:+ await _record_turn_resumed(organization_id, workflow_copilot_chat_id, turn_id)+ async with cache.get_lock(_credential_pause_lock_key(organization_id, workflow_copilot_chat_id, turn_id)):+ record = _decode_active_pause(await cache.get(active_key))+ if record is None or record.status != "resolving" or record.claim_id != claim.claim_id:+ return None+ ttl = _longest_credential_pause_record_ttl()+ if credential_id is None:+ registration = claim.registration.model_copy(update={"outcome": outcome})+ reopened = _extend_pause(+ replace(record, status="pending", claim_id=None, claim_deadline=None, registration=registration),+ settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS,+ )+ await cache.set(active_key, _encode_record(reopened), ex=ttl)+ return reopened.expires_at+ consumed = _ActivePauseRecord(+ record.resume_token, "consumed", record.expires_at, generated_credential_id=credential_id+ )+ await cache.set(active_key, _encode_record(consumed), ex=ttl)+ await cache.set(+ credential_response_cache_key(organization_id, workflow_copilot_chat_id, turn_id),+ encode_credential_response("connected", credential_id, record.resume_token),+ ex=ttl,+ )+ return record.expires_at++def credential_pause_reason(ctx: Any) -> str | None:"""Typed-signal-only detector for a mid-build credential ask.@@ -880,6 +992,7 @@ async def _run_credential_pause(admit_connected: Callable[[Credential], Awaitable[bool]] | None = None,allow_second_ask: bool = False,anchor_tool_call_id: str | None = None,+ registration: WorkflowCopilotCredentialRegistration | None = None,) -> CredentialPauseResolution | None:"""Send the credential card and wait for the user's decision.@@ -889,6 +1002,7 @@ async def _run_credential_pause(Recovery-capable clients restore the same active card through chat history."""update_ask = update_credential_id is not None+ ctx.credential_registration_outcome = Noneif not credential_pause_transport_ready(ctx, copilot_config, allow_second_ask=update_ask or allow_second_ask):return None# Latch before async checks so a declined transport cannot trigger another pause. Only the pick ask@@ -931,9 +1045,9 @@ async def _run_credential_pause(turn_id = ctx.turn_id or ""resume_token = _new_resume_token()expires_at = now + timedelta(seconds=timeout_seconds)- manual_sign_in = None if update_ask else _manual_sign_in_offer(ctx, login_page_urls)- if manual_sign_in is not None:- # Signing in takes minutes in the same tab, so the card must survive a reload.+ manual_sign_in = None if update_ask or registration else _manual_sign_in_offer(ctx, login_page_urls)+ if manual_sign_in is not None or registration is not None:+ # Signing in or saving a generated credential can outlast the tab, so the card must survive a reload.ctx.credential_recovery_armed = Truerecovery_enabled = _credential_recovery_enabled(ctx)# Establish the active-pause record before the frame carries the token: the@@ -953,6 +1067,7 @@ async def _run_credential_pause(expires_at=expires_at,anchor_tool_call_id=anchor_tool_call_id,sign_in_browser_session_id=manual_sign_in.browser_session_id if manual_sign_in else None,+ registration=registration,timestamp=now,)await cache.set(@@ -963,8 +1078,11 @@ async def _run_credential_pause(card=card if recovery_enabled else None,recovery_token_digest=ctx.credential_recovery_token_digest if recovery_enabled else None,manual_sign_in=manual_sign_in,+ registration=registration,),- ex=_longest_credential_pause_record_ttl() if manual_sign_in else _credential_pause_record_ttl(timeout_seconds),+ ex=_longest_credential_pause_record_ttl()+ if manual_sign_in or registration+ else _credential_pause_record_ttl(timeout_seconds),)await stream.send(card)@@ -991,9 +1109,16 @@ async def _run_credential_pause(return "live"await cache.set(active_key,- _encode_active_pause(resume_token, expires_at, consumed=True),+ _encode_active_pause(+ resume_token,+ expires_at,+ consumed=True,+ registration=record.registration+ if record is not None and record.resume_token == resume_token+ else None,+ ),ex=_longest_credential_pause_record_ttl()- if manual_sign_in+ if manual_sign_in or registrationelse _credential_pause_record_ttl(timeout_seconds),)return None@@ -1060,6 +1185,23 @@ async def _run_credential_pause(if resolution is None and not invalidated:invalidated_resolution = await _invalidate_active_pause_record()resolution = invalidated_resolution if isinstance(invalidated_resolution, CredentialPauseResolution) else None+ generated_credential_id: str | None = None+ if registration is not None:+ settled_record = _decode_active_pause(+ await cache.get(credential_pause_active_key(organization_id, chat_id, turn_id))+ )+ if settled_record is None or settled_record.resume_token != resume_token:+ settled_record = None+ settled_registration = settled_record.registration if settled_record is not None else None+ generated_credential_id = settled_record.generated_credential_id if settled_record is not None else None+ if settled_registration is not None:+ ctx.credential_registration_outcome = settled_registration.outcome+ # A create claimed but never finished (the pause ran out mid-create) may still have saved a credential.+ connected = resolution is not None and resolution.action == "connected"+ if settled_registration.attempted and settled_registration.outcome is None and not connected:+ ctx.credential_registration_outcome = "unknown"+ if generated_credential_id is not None or ctx.credential_registration_outcome == "unknown":+ ctx.credential_generation_spent = Trueif resolution is None:settle("timeout")return None@@ -1095,7 +1237,9 @@ async def _run_credential_pause(return Noneawait answered("connected", credential)return resolution- if admit_connected is not None and not await admit_connected(credential):+ # A credential this card just generated has no saved site yet; the card's own origin is its only placement.+ is_generated = credential.credential_id == generated_credential_id+ if admit_connected is not None and not is_generated and not await admit_connected(credential):ctx.credential_pause_outcome = "not_admitted"await answered("not_admitted")return resolution@@ -1103,7 +1247,7 @@ async def _run_credential_pause(_apply_connected_credential_to_policy(ctx, policy, credential)ctx.credential_pause_outcome = "connected"await answered("connected", credential)- return resolution+ return replace(resolution, generated=True) if is_generated else resolutiondef arm_credential_pause_gate(ctx: Any) -> None:@@ -1135,6 +1279,7 @@ async def request_credential_pause(admit_connected: Callable[[Credential], Awaitable[bool]] | None = None,allow_second_ask: bool = False,anchor_tool_call_id: str | None = None,+ registration: WorkflowCopilotCredentialRegistration | None = None,) -> CredentialPauseResolution | None:"""Raise the card from the model's own ``request_credential`` call and wait, inline, for theanswer, so tool calls the model issued alongside it can await ``credential_pause_settled``."""@@ -1149,12 +1294,17 @@ async def request_credential_pause(message,stream,copilot_config,- reason=update_reason if update_ask else "login_credentials_unresolved",+ reason=update_reason+ if update_ask+ else "credential_registration"+ if registration+ else "login_credentials_unresolved",login_page_urls=[login_page_url],update_credential_id=update_credential_id,admit_connected=admit_connected,allow_second_ask=allow_second_ask,anchor_tool_call_id=anchor_tool_call_id,+ registration=None if update_ask else registration,)if not update_ask:ctx.credential_pause_reaskable_by_run = resolution is None or resolution.action == "skip"diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 1d53d9c8d..dc1c334aa 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -139,6 +139,11 @@ FINAL_REPLY_OBSERVATION = (NUDGE_SENTINEL + "Your last response was empty. Tools are unavailable for this one response; ""reply to the user now with your final response for this turn.")+RAW_SECRET_REPLY_WITHHELD_OBSERVATION = (+ NUDGE_SENTINEL + "Your last reply was not shown to the user. The output check withheld it because it contained "+ "text shaped like a raw secret: a value written after a password, passcode, secret, API key, or token label. "+ "Tools are unavailable for this one response; reply to the user now with your final response for this turn."+)TOKEN_BUDGET = DEFAULT_TOKEN_BUDGETSYNTHESIZED_BLOCK_PERSISTENCE_TOOL = "update_and_run_blocks"# Both tools re-author the workflow draft and clear the coverage-reopen flag; the steer must fire@@ -1418,6 +1423,7 @@ async def run_final_reply_drain(session: Session | None,hooks: FinalReplyRunHooks,run_config: RunConfig,+ observation: str = FINAL_REPLY_OBSERVATION,) -> RunResultStreaming:reply_run_config = replace(run_config,@@ -1429,7 +1435,7 @@ async def run_final_reply_drain(try:return await _run_streamed_with_deadline(agent,- FINAL_REPLY_OBSERVATION,+ observation,ctx,session,_SendTrackingStream(stream),diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex 3be1aca51..60dcbd1cd 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -86,6 +86,7 @@ from skyvern.forge.sdk.copilot.workflow_yaml import (stored_block_code,stored_workflow_yaml,)+from skyvern.forge.sdk.schemas.workflow_copilot import CredentialRegistrationfrom skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinitionfrom skyvern.utils.yaml_loader import dump_workflow_yaml@@ -219,12 +220,10 @@ from .mcp_hooks import get_skyvern_mcp_alias_map as get_skyvern_mcp_alias_mapfrom .page_challenge import FRESH_BROWSER_TOOL_NAME, SOLVE_TOOL_NAME, solve_page_challenge, start_fresh_browserfrom .page_observation import _record_composition_page_observation as _record_composition_page_observationfrom .page_observation import _resolve_url_title as _resolve_url_title-from .run_execution import RUN_BLOCKS_STAGNATION_WINDOW_SECONDS as RUN_BLOCKS_STAGNATION_WINDOW_SECONDSfrom .run_execution import (RUN_RESULTS_MAX_ROW_KEYS,)from .run_execution import WatchdogExitReason as WatchdogExitReason-from .run_execution import _any_quiet_block_requested as _any_quiet_block_requestedfrom .run_execution import _attach_action_traces as _attach_action_tracesfrom .run_execution import _block_end_urls_by_label as _block_end_urls_by_labelfrom .run_execution import _cancel_run_task_if_not_final as _cancel_run_task_if_not_final@@ -238,7 +237,6 @@ from .run_execution import (_diagnosis_repair_tool_error,_get_run_results,)-from .run_execution import _progress_marker as _progress_markerfrom .run_execution import _read_progress_sources as _read_progress_sourcesfrom .run_execution import _record_diagnosis_repair_contract as _record_diagnosis_repair_contractfrom .run_execution import _record_run_blocks_result as _record_run_blocks_result@@ -995,6 +993,7 @@ async def request_credential_tool(reason: str,credential_id: str | None = None,rejected_by_site: bool = False,+ registration: CredentialRegistration | None = None,) -> str:"""Ask the user, in chat, how to sign in to a sign-in page: add or pick a saved credential, orsign in themselves in the live browser and save that sign-in as a browser profile.@@ -1011,6 +1010,14 @@ async def request_credential_tool(refused that bound credential's saved password or code; a failed run's `credential_update` levernames the one credential bound to its failed block. The card asks the user to update it andcomes back `updated` once they save; re-run the sign-in then.+ When the user asked you to create one new account and the sign-up form needs a password, pass+ `registration` with the sign-up page as `login_page_url`: the card offers to generate a password+ and save it with that username and credential name, and you only ever get the credential's id.+ `password_length` is 24 to 128; a longer or shorter site rule comes back `unsupported_constraints`.+ `charset` is `alphanumeric` or `alphanumeric_symbols`; when the site's rules fit neither, call+ without `registration` so the user adds the login. Never write a password yourself.+ A second `registration` ask on a turn that already generated a credential, or may have, comes+ back `already_generated` with no card.The call waits for the user's answer and comes back `connected` with the credentialto bind, `skipped`, `unanswered`, or `unavailable` — follow the `next` or `fallback` itcarries. It comes back `signed_in` when the user signed in themselves in the live browser:@@ -1023,6 +1030,7 @@ async def request_credential_tool("reason": reason,"credential_id": credential_id,"rejected_by_site": rejected_by_site,+ "registration": registration.model_dump() if registration else None,}result: dict[str, Any] = {}try:@@ -1037,6 +1045,7 @@ async def request_credential_tool(credential_id,rejected_by_site,anchor_tool_call_id=_originating_call_id(ctx),+ registration=registration,)finally:# A card on screen right now owns the gate; this call must not open it for one it never@@ -1371,6 +1380,9 @@ async def get_run_results_tool(returned run, excluding runs started by any Copilot chat, newest first; itholds at most 5, so more may exist. When that lookup fails the result hasnewer_finished_runs_unavailable instead, so a missing list is not "none".+ execution_source with source_kind run_version describes the saved version the run executed, which may+ differ from the current draft (current_draft_code_matches compares each code block); disposition+ unavailable means that version cannot be proven, so no executed source is given."""copilot_ctx = ctx.contextparams: dict[str, Any] = {}diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.pyindex 9e3d280ef..fe9b1dca9 100644--- a/skyvern/forge/sdk/copilot/tools/_shared.py+++ b/skyvern/forge/sdk/copilot/tools/_shared.py@@ -120,9 +120,8 @@ async def _call_internal_browser_tool(_OUTCOME_EVIDENCE_BLOCK_TYPES = frozenset({BlockType.EXTRACTION.value, BlockType.VALIDATION.value})-# Absolute upper bound on a single ``run_blocks`` tool invocation. Exists only-# as a last-resort trip wire for runaway loops — progressing runs should never-# approach this. The OpenAI Agents SDK wraps the tool in+# Absolute upper bound on a single ``run_blocks`` tool invocation, including a run+# whose rows have gone silent. The OpenAI Agents SDK wraps the tool in# ``asyncio.wait_for(..., timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS)``; the# inner poll loop leaves a 10 s headroom below this ceiling for orderly# cleanup before the SDK cancels.diff --git a/skyvern/forge/sdk/copilot/tools/credential_fill.py b/skyvern/forge/sdk/copilot/tools/credential_fill.pyindex 5f0f1b571..72a0ecfc4 100644--- a/skyvern/forge/sdk/copilot/tools/credential_fill.py+++ b/skyvern/forge/sdk/copilot/tools/credential_fill.py@@ -25,6 +25,10 @@ from skyvern.forge.sdk.copilot.blocker_signal import (from skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.credential_fill_fields import CREDENTIAL_FILL_FIELDS+from skyvern.forge.sdk.copilot.credential_generation import (+ REGISTRATION_PASSWORD_MAX_LENGTH,+ REGISTRATION_PASSWORD_MIN_LENGTH,+)from skyvern.forge.sdk.copilot.credential_pause import (RAW_SECRET_CONNECTED_NEXT,CredentialPauseResolution,@@ -76,6 +80,7 @@ from skyvern.forge.sdk.schemas.credentials import (PasswordCredential,TotpType,)+from skyvern.forge.sdk.schemas.workflow_copilot import CredentialRegistration, WorkflowCopilotCredentialRegistrationfrom skyvern.forge.sdk.services.credentials import generate_totp_code, normalize_totp_configfrom skyvern.webeye.utils.dom import is_post_dispatch_click_timeout@@ -490,6 +495,40 @@ _CREDENTIAL_CARD_FALLBACK = ("Ask the user in prose to add the login on the Credentials page and reply with its exact saved name.")+_REGISTRATION_OUTCOME_NOTES: dict[str, dict[str, str]] = {+ "rejected": {+ "registration_outcome": "rejected",+ "registration_detail": (+ "Generate and save saved nothing: validation or the vault refused it, or the pre-check timed out."+ ),+ },+ "unknown": {+ "registration_outcome": "unknown",+ "registration_detail": "Generate and save was not confirmed, so a credential under this name may exist.",+ },+}+++def _registration_ask_error(+ registration: CredentialRegistration, policy: RequestPolicy, credential_id: str | None+) -> dict[str, Any] | None:+ if credential_id:+ return {"ok": False, "error": "Pass either `registration` or `credential_id`, not both."}+ if policy.raw_secret_detected:+ return {"ok": False, "error": "Generating a credential is unavailable on a turn that contains a secret."}+ if not registration.username.strip() or not registration.credential_name.strip():+ return {"ok": False, "error": "`registration` needs a non-empty `username` and `credential_name`."}+ if not REGISTRATION_PASSWORD_MIN_LENGTH <= registration.password_length <= REGISTRATION_PASSWORD_MAX_LENGTH:+ return {+ "ok": True,+ "status": "unsupported_constraints",+ "detail": (+ f"Generated passwords are {REGISTRATION_PASSWORD_MIN_LENGTH} to "+ f"{REGISTRATION_PASSWORD_MAX_LENGTH} characters, so no card was shown."+ ),+ }+ return None+def _missing_totp_card_fallback(credential_name: str) -> str:return (@@ -590,6 +629,7 @@ async def _request_credential(credential_id: str | None = None,rejected_by_site: bool = False,anchor_tool_call_id: str | None = None,+ registration: CredentialRegistration | None = None,) -> dict[str, Any]:policy = copilot_ctx.request_policyif not isinstance(policy, RequestPolicy) or (policy.raw_secret_detected and not policy.raw_secret_redacted_draft):@@ -597,6 +637,29 @@ async def _request_credential(ask_origin = canonicalize_origin(login_page_url)if not is_resolved_page_url(login_page_url) or ask_origin is None:return {"ok": False, "error": "Provide the absolute HTTP(S) sign-in page URL for the credential card."}+ card_registration: WorkflowCopilotCredentialRegistration | None = None+ if registration is not None:+ registration_error = _registration_ask_error(registration, policy, credential_id)+ if registration_error is not None:+ return registration_error+ if copilot_ctx.credential_generation_spent:+ return {+ "ok": True,+ "status": "already_generated",+ "detail": "Generate and save already ran on this turn, so no second generate card was shown.",+ }+ if not copilot_ctx.client_supports_credential_generation:+ return {+ "ok": True,+ "status": "unavailable",+ "detail": "Generating a credential from the card is not available on this turn.",+ }+ card_registration = WorkflowCopilotCredentialRegistration(+ username=registration.username.strip(),+ credential_name=registration.credential_name.strip(),+ password_length=registration.password_length,+ charset=registration.charset,+ )if policy.raw_secret_redacted_draft:if credential_id:return {@@ -694,6 +757,7 @@ async def _request_credential(admit_connected=admit_connected,allow_second_ask=handback,anchor_tool_call_id=anchor_tool_call_id,+ registration=card_registration,)except BaseException:if recovery is not None:@@ -726,16 +790,21 @@ async def _request_credential(clear_tool_blocker_signals_for_reason_codes(copilot_ctx, frozenset({CREDENTIAL_ORIGIN_RECOVERY_PENDING_REASON_CODE}))+ registration_outcome = copilot_ctx.credential_registration_outcome if card_registration is not None else None+ registration_note = _REGISTRATION_OUTCOME_NOTES[registration_outcome] if registration_outcome is not None else {}if resolution is None:- return {+ unanswered: dict[str, Any] = {"ok": True,"status": "unanswered","outcome": copilot_ctx.credential_pause_outcome or "timeout",- "next": (+ **registration_note,+ }+ if card_registration is None:+ unanswered["next"] = ("The user did not answer the card. Continue without the credential: keep the credential ""parameter placeholder in the draft and do not ask again this turn."- ),- }+ )+ return unansweredif resolution.signed_in is not None:return {"ok": True,@@ -756,26 +825,31 @@ async def _request_credential(),}if credential is None:- return {- "ok": True,- "status": "skipped",- "next": (+ skipped: dict[str, Any] = {"ok": True, "status": "skipped", **registration_note}+ if card_registration is None:+ skipped["next"] = ("The user chose not to connect a credential now. Keep the credential parameter placeholder ""in the draft, do not ask again this turn, and say a test run may stop at the login step."- ),- }- return {+ )+ return skipped+ connected: dict[str, Any] = {"ok": True,"status": "connected","credential_id": credential.credential_id,"credential_name": credential.name,- "next": (- RAW_SECRET_CONNECTED_NEXT- if policy.raw_secret_redacted_draft- else "Bind this credential as the workflow's credential parameter and continue the build; run the "- "blocks that were waiting on the login."- ),}+ if policy.raw_secret_redacted_draft:+ connected["next"] = RAW_SECRET_CONNECTED_NEXT+ elif card_registration is not None:+ if resolution.generated:+ connected["generated"] = True+ connected["site_effect"] = "none"+ else:+ connected["next"] = (+ "Bind this credential as the workflow's credential parameter and continue the build; run the "+ "blocks that were waiting on the login."+ )+ return connectedasync def _missing_totp_ask_outcome(diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 8fa109ab5..509aad3c9 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -228,7 +228,11 @@ from skyvern.forge.sdk.copilot.turn_halt import (stash_turn_halt_from_blocker_signal,)from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin-from skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml, runner_code_block_associations+from skyvern.forge.sdk.copilot.workflow_yaml import (+ _process_workflow_yaml,+ runner_code_block_associations,+ stored_block_code,+)from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.db.enums import BrowserSeedSourcefrom skyvern.forge.sdk.executor.factory import AsyncExecutorFactory@@ -268,6 +272,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (WorkflowRunOutputParameter,WorkflowRunParameter,WorkflowRunStatus,+ workflow_definition_sha256,)from skyvern.forge.sdk.workflow.runtime_completion import contract_from_request_criteriafrom skyvern.forge.sdk.workflow.runtime_secret_bridge import consume_copilot_runtime_secret_values@@ -356,19 +361,16 @@ _POST_RUN_PAGE_PARSE_TIMEOUT_SECONDS = 15.0# starts with a ZIP local/central/spanning signature, so treat these prefixes as fail-closed._ZIP_MAGIC_PREFIXES = (b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08")-# Primary exit condition: seconds of no observed progress across the combined-# run / block / step heartbeat. Sized to accommodate the slowest single LLM-# round-trip (~30-60 s in practice) with headroom; going tighter risks-# false-positives on healthy runs.-RUN_BLOCKS_STAGNATION_WINDOW_SECONDS = 90---# 5 s balances responsiveness (18 samples inside the stagnation window) against-# DB load (240 polls worst case at the safety ceiling).+# 5 s balances how quickly a terminal row is noticed against DB load (240 polls worst case at the safety ceiling).RUN_BLOCKS_POLL_INTERVAL_SECONDS = 5.0COPILOT_SANDBOX_UNAVAILABLE_ERROR = "Sandboxed worker is unavailable; execution was not started."COPILOT_SNAPSHOT_PREPARATION_ERROR = "Unable to prepare the Copilot test-run snapshot; execution was not started."+COPILOT_PROPOSAL_CHANGED_ERROR = "The pending Copilot proposal changed before the test started; reload and try again."+COPILOT_PROPOSAL_SETTINGS_CHANGED_ERROR = (+ "This request's workflow settings differ from the pending Copilot proposal's; "+ "re-save the proposal with the current settings, then test again."+)# Block types that can reach exec() in the API process, so a run containing one may# only proceed on the sandboxed worker. CODE compiles user code directly;@@ -1264,128 +1266,42 @@ def _failure_action_trace_summary(failed_result: Mapping[str, Any] | None) -> lireturn _retained_action_observations([failed_result])-# Watchdog exit reasons. ``success`` means the run reached a trustworthy-# terminal status inside the poll loop OR after the post-drain reconcile.-# The run-ending reasons share the reconcile path but produce distinct-# error messages: ``stagnation`` is the primary trip (no progress signals-# for ``RUN_BLOCKS_STAGNATION_WINDOW_SECONDS`` seconds), ``ceiling`` is the-# last-resort budget-exhausted branch, and ``task_exit_unfinalized`` is the-# rare race where ``execute_workflow`` naturally exits before writing a-# terminal row. ``paused`` is the exception: the run is alive and waiting on a-# person, so it is neither cancelled nor reconciled.+# ``success`` means a trustworthy terminal status was read in the poll loop or after the post-drain+# reconcile; ``paused`` leaves a live run waiting on a person, so it is neither cancelled nor reconciled.WatchdogExitReason = Literal["success",- "stagnation","ceiling","task_exit_unfinalized","paused",]-# Block types that legitimately execute long silent periods: one DB write on-# entry, work done without intermediate writes (sleep / LLM call / await human-# input / browser download wait), one write on finish. The watchdog can't-# distinguish these from "stuck", so any invocation that includes one disables-# stagnation for the whole run and relies on the safety ceiling alone.-_QUIET_BLOCK_TYPES: frozenset[str] = frozenset(- {- BlockType.WAIT.value,- BlockType.TEXT_PROMPT.value,- BlockType.WEB_SEARCH.value,- BlockType.HUMAN_INTERACTION.value,- BlockType.FILE_DOWNLOAD.value,- BlockType.FILE_UPLOAD.value,- # A code block writes its row on entry and exit and nothing in between, so a long- # login or wait inside one is indistinguishable from a frozen run.- BlockType.CODE.value,- }-)---def _any_quiet_block_requested(- copilot_ctx: CopilotContext,- labels: list[str] | None,- *,- workflow: Workflow | None = None,-) -> bool:- """Return True if a selected block or its descendants can run without DB updates.-- Use the execution snapshot when supplied, including staged edits.- """- if not labels:- return False- last_workflow = workflow if workflow is not None else getattr(copilot_ctx, "last_workflow", None)- if last_workflow is None:- return False- definition = getattr(last_workflow, "workflow_definition", None)- pending = [(block, False) for block in (getattr(definition, "blocks", None) or [])]- selected_labels = set(labels)- while pending:- block, parent_selected = pending.pop()- selected = parent_selected or getattr(block, "label", None) in selected_labels- pending.extend((child, selected) for child in _typed_child_blocks(block))- if not selected:- continue- block_type = getattr(block, "block_type", None)- if block_type is None:- continue- block_type_str = block_type.value if hasattr(block_type, "value") else str(block_type)- if block_type_str in _QUIET_BLOCK_TYPES:- return True- return False--async def _read_progress_sources(ctx: CopilotContext,workflow_run_id: str,-) -> tuple[WorkflowRun | None, datetime | None, datetime | None]:- """Read one ``workflow_runs`` row + the two progress aggregates needed- by the watchdog marker. Three cheap indexed queries; no row hydration- on the aggregate side. The two repo calls run concurrently — they open- separate async sessions and hit different tables.- """+) -> tuple[WorkflowRun | None, datetime | None]:+ """Read one ``workflow_runs`` row and the latest block progress timestamp. The two repo calls run+ concurrently because they open separate async sessions and hit different tables."""- async def _read_timestamps() -> tuple[datetime | None, datetime | None]:+ async def _read_block_timestamp() -> datetime | None:try:- return await app.DATABASE.tasks.get_workflow_run_progress_timestamps(+ return await app.DATABASE.tasks.get_workflow_run_block_progress_timestamp(workflow_run_id=workflow_run_id,organization_id=ctx.organization_id,)except Exception:LOG.warning(- "Workflow run progress timestamps read failed",+ "Workflow run block progress timestamp read failed",workflow_run_id=workflow_run_id,exc_info=True,)- return None, None+ return None- run, (step_ts, block_ts) = await asyncio.gather(+ run, block_ts = await asyncio.gather(_safe_read_workflow_run(workflow_run_id, ctx.organization_id, context="watchdog-poll"),- _read_timestamps(),- )- return run, step_ts, block_ts---def _progress_marker(- run: WorkflowRun | None,- step_ts: datetime | None,- block_ts: datetime | None,-) -> tuple[Any, ...]:- """Hashable scalar snapshot. Changes iff any observable progress has- occurred at the run, step, or block level since the last poll. Every- ``update_step`` fires during action execution (including incremental- token/cost accumulators at ``forge/agent.py:1449``), so- ``max(step.modified_at)`` is the per-LLM-call heartbeat. Non-task blocks- (CODE, TEXT_PROMPT) don't create step rows — ``max(workflow_run_block.modified_at)``- covers that case. ``run.modified_at`` and ``run.status`` catch the- run-level transitions that happen outside those two tables.- """- return (- run.status if run else None,- run.modified_at if run is not None else None,- step_ts,- block_ts,+ _read_block_timestamp(),)+ return run, block_tsasync def _watchdog_error_message(@@ -1406,15 +1322,7 @@ async def _watchdog_error_message(ctx.origin_run_redaction_registry,workflow_run_id,)- if exit_reason == "stagnation":- body = (- f"The run has not made progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s. "- f"No step, block, or workflow-run row updates were observed in that window. "- f"The page is most likely blocked by a captcha, popup, anti-bot challenge, "- f"hidden validation error, or an infinite-retry loop on an action the agent "- f"cannot detect is failing."- )- elif exit_reason == "paused":+ if exit_reason == "paused":# A pause is a healthy waiting state rather than an uncertain outcome, so it returns here# instead of picking up the shared "outcome is uncertain, do not re-invoke" tail below. This# is the one arm that directs the model to relay its own text, so it carries no run id.@@ -1426,9 +1334,8 @@ async def _watchdog_error_message()elif exit_reason == "ceiling":body = (- f"The run exceeded the {budget_seconds}s absolute ceiling "- f"while still showing progress. The workflow is too long to fit in a single "- f"tool invocation — split it into smaller block groups."+ f"The run did not reach a terminal status within the {budget_seconds}s absolute ceiling. "+ f"If the workflow is too long to fit in a single tool invocation, split it into smaller block groups.")else: # task_exit_unfinalizedlast_observed = f"last observed status: {run.status}" if run is not None else "workflow run row was unreadable"@@ -1453,12 +1360,10 @@ def _watchdog_user_facing_summary(budget_seconds: int,run: WorkflowRun | None,) -> str:- if exit_reason == "stagnation":- return f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s."if exit_reason == "paused":return "The run is paused, waiting for a person to approve or reject it."if exit_reason == "ceiling":- return f"The run exceeded the {budget_seconds}s absolute ceiling while still showing progress."+ return f"The run did not finish within the {budget_seconds}s absolute ceiling."if run is not None:return f"The run ended before recording a trustworthy terminal status. Last observed status: {run.status}."return "The run ended before recording a trustworthy terminal status."@@ -1794,6 +1699,182 @@ def _execution_source_facts(}+RunVersionUnavailableReason = Literal["version_missing", "identity_mismatch", "overwritten_after_run_start"]+++class _RunVersionSource(TypedDict):+ """The saved version a finished run executed, read from the run's own workflow row."""++ source_kind: Literal["run_version"]+ disposition: Literal["available", "unavailable"]+ workflow_run_id: str+ workflow_id: str | None+ reason: NotRequired[RunVersionUnavailableReason]+ workflow_version: NotRequired[int]+ failed_block_label: NotRequired[str]+ failed_statement: NotRequired[str]+ failed_statement_truncated: NotRequired[bool]+ declared_parameter_keys: NotRequired[list[str]]+ declared_parameter_keys_omitted: NotRequired[int]+ block_code_sha256: NotRequired[dict[str, str]]+ current_draft_code_matches: NotRequired[dict[str, bool]]+ code_block_labels_omitted: NotRequired[int]+ failed_block_code: NotRequired[str]+ failed_block_code_truncated: NotRequired[bool]+++_RUN_VERSION_SOURCE_MAX_CHARS = 8000+_RUN_VERSION_STATEMENT_MAX_CHARS = 600+_RUN_VERSION_IDENTIFIER_MAX_CHARS = 120+# Keys and labels stop this far short of the limit, so their omission counts and the code's truncation flag still fit.+_RUN_VERSION_MARKER_RESERVE_CHARS = 256+++def _is_run_version_result(data: Mapping[str, Any]) -> bool:+ source = data.get("execution_source")+ return isinstance(source, Mapping) and source.get("source_kind") == "run_version"+++def _run_version_rewritten_after_start(run_workflow: Workflow, run_created_at: datetime) -> bool:+ # A saved version can be overwritten in place after the run; a test version is only ever soft-deleted.+ return run_workflow.created_by != COPILOT_TEST_WORKFLOW_CREATOR and _as_utc(run_workflow.modified_at) > _as_utc(+ run_created_at+ )+++def _run_version_unproven_reason(+ ctx: CopilotContext, run: WorkflowRun, run_workflow: Workflow | None+) -> RunVersionUnavailableReason | None:+ if run_workflow is None:+ return "version_missing"+ if run_workflow.workflow_id != run.workflow_id or run_workflow.workflow_permanent_id != ctx.workflow_permanent_id:+ return "identity_mismatch"+ if run.workflow_definition_sha256 is not None:+ current = workflow_definition_sha256(run_workflow.workflow_definition)+ return None if current == run.workflow_definition_sha256 else "overwritten_after_run_start"+ if _run_version_rewritten_after_start(run_workflow, run.created_at):+ return "overwritten_after_run_start"+ return None+++def _cut_to_serialized_chars(text: str, limit: int) -> str:+ """Longest prefix whose JSON form fits ``limit``; escaping makes one character cost up to 12."""+ low, high = 0, len(text)+ while low < high:+ middle = (low + high + 1) // 2+ if len(json.dumps(text[:middle])) - 2 <= limit:+ low = middle+ else:+ high = middle - 1+ return text[:low]+++def _prompt_safe_text(ctx: CopilotContext, text: str) -> str:+ """Exact-match scrubbing has to see the whole string, so this runs before any cut."""+ return redact_raw_secrets_for_prompt(scrub_secrets_from_structure(ctx, text))+++def _run_version_source(+ ctx: CopilotContext,+ run: WorkflowRun,+ run_workflow: Workflow | None,+ *,+ failed_block_label: str | None,+ failing_line: int | None,+ requested_block_labels: Sequence[str],+) -> _RunVersionSource:+ receipt: _RunVersionSource = {+ "source_kind": "run_version",+ "disposition": "unavailable",+ "workflow_run_id": run.workflow_run_id,+ "workflow_id": run.workflow_id,+ }+ reason = _run_version_unproven_reason(ctx, run, run_workflow)+ if reason is not None or run_workflow is None:+ receipt["reason"] = reason or "version_missing"+ return receipt+ receipt["disposition"] = "available"+ receipt["workflow_version"] = run_workflow.version++ code_inputs = _selected_code_security_inputs(+ _workflow_definition_blocks_for_code_security(run_workflow.workflow_definition),+ selected_labels=set(),+ include_descendants=True,+ )+ safe_code_by_label = {code_input.label: _prompt_safe_text(ctx, code_input.code) for code_input in code_inputs}+ raw_failed_code = next(+ (code_input.code for code_input in code_inputs if code_input.label == failed_block_label), ""+ )+ safe_failed_code = safe_code_by_label.get(failed_block_label) if failed_block_label else None+ if failed_block_label:+ receipt["failed_block_label"] = _cut_to_serialized_chars(+ _prompt_safe_text(ctx, failed_block_label), _RUN_VERSION_IDENTIFIER_MAX_CHARS+ )+ failed_lines = safe_failed_code.splitlines() if safe_failed_code else []+ # Scrubbing a multi-line secret shifts every later line, so the recorded line number would name the wrong statement.+ lines_kept_positions = len(failed_lines) == len(raw_failed_code.splitlines())+ if failing_line is not None and lines_kept_positions and 0 < failing_line <= len(failed_lines):+ statement = failed_lines[failing_line - 1].strip()+ receipt["failed_statement"] = _cut_to_serialized_chars(statement, _RUN_VERSION_STATEMENT_MAX_CHARS)+ if receipt["failed_statement"] != statement:+ receipt["failed_statement_truncated"] = True++ declared_keys = [+ _cut_to_serialized_chars(_prompt_safe_text(ctx, parameter.key), _RUN_VERSION_IDENTIFIER_MAX_CHARS)+ for parameter in _workflow_parameters(run_workflow)+ if not isinstance(parameter, OutputParameter)+ ]+ receipt["declared_parameter_keys"] = []+ for index, key in enumerate(declared_keys):+ receipt["declared_parameter_keys"].append(key)+ if len(json.dumps(receipt)) > _RUN_VERSION_SOURCE_MAX_CHARS - _RUN_VERSION_MARKER_RESERVE_CHARS:+ receipt["declared_parameter_keys"].pop()+ receipt["declared_parameter_keys_omitted"] = len(declared_keys) - index+ break++ draft_yaml = ctx.staged_workflow_yaml or ctx.workflow_yaml or ""+ code_labels = [label for label in dict.fromkeys(requested_block_labels) if label in safe_code_by_label]+ receipt["block_code_sha256"] = {}+ receipt["current_draft_code_matches"] = {}+ labels_omitted = 0+ for index, label in enumerate(code_labels):+ # Hashing scrubbed text keeps the digest from confirming a guessed secret next to the masked code.+ digest = hashlib.sha256(safe_code_by_label[label].encode("utf-8")).hexdigest()+ draft_code = stored_block_code(draft_yaml, label, allow_empty=True)+ shown_label = _cut_to_serialized_chars(_prompt_safe_text(ctx, label), _RUN_VERSION_IDENTIFIER_MAX_CHARS)+ if shown_label in receipt["block_code_sha256"]:+ # Distinct labels can share a cut or scrubbed form; keep the first rather than merge their facts.+ labels_omitted += 1+ continue+ receipt["block_code_sha256"][shown_label] = digest+ receipt["current_draft_code_matches"][shown_label] = (+ draft_code is not None+ and hashlib.sha256(_prompt_safe_text(ctx, draft_code).encode("utf-8")).hexdigest() == digest+ )+ if len(json.dumps(receipt)) > _RUN_VERSION_SOURCE_MAX_CHARS - _RUN_VERSION_MARKER_RESERVE_CHARS:+ del receipt["block_code_sha256"][shown_label]+ del receipt["current_draft_code_matches"][shown_label]+ labels_omitted += len(code_labels) - index+ break+ if labels_omitted:+ receipt["code_block_labels_omitted"] = labels_omitted++ if safe_failed_code:+ receipt["failed_block_code"] = safe_failed_code+ if len(json.dumps(receipt)) > _RUN_VERSION_SOURCE_MAX_CHARS:+ receipt["failed_block_code_truncated"] = True+ low, high = 0, len(safe_failed_code)+ while low < high:+ middle = (low + high + 1) // 2+ receipt["failed_block_code"] = safe_failed_code[:middle]+ if len(json.dumps(receipt)) <= _RUN_VERSION_SOURCE_MAX_CHARS:+ low = middle+ else:+ high = middle - 1+ receipt["failed_block_code"] = safe_failed_code[:low]+ return receipt++_LOOP_INPUTS_MAX_LOOPS = 20_LOOP_VALUES_COUNTS_MAX_ROWS = 10@@ -1911,16 +1992,12 @@ def _attach_loop_inputs(def _read_run_loop_inputs(- run_workflow: Workflow | None, run_created_at: datetime, workflow_run_id: str, rows: Sequence[WorkflowRunBlock]+ run_workflow: Workflow | None, workflow_run_id: str, rows: Sequence[WorkflowRunBlock]) -> tuple[LoopInputs | LoopInputsUnavailable | None, list[str]]:- loop_definitions = _loop_definitions_or_unavailable(run_workflow) if run_workflow is not None else None- # A saved version can be overwritten in place after the run; a test version is only ever soft-deleted.- if run_workflow is None or (- run_workflow.created_by != COPILOT_TEST_WORKFLOW_CREATOR- and _as_utc(run_workflow.modified_at) > _as_utc(run_created_at)- ):- ran_loop = loop_definitions is not None or any(row.block_type == BlockType.FOR_LOOP for row in rows)+ if run_workflow is None:+ ran_loop = any(row.block_type == BlockType.FOR_LOOP for row in rows)return (LoopInputsUnavailable() if ran_loop else None), []+ loop_definitions = _loop_definitions_or_unavailable(run_workflow)if loop_definitions is None:return None, []return _observed_loop_inputs(loop_definitions, workflow_run_id, rows)@@ -3699,6 +3776,14 @@ async def _run_blocks_and_collect_debug(reused_origin_output_labels=reused_origin_output_labels,origin_workflow_run_id=ctx.repair_origin_outputs_run_id if reused_origin_output_labels else None,)+ if ctx.settings_diverged_proposal_token == (execution.proposal_owner_turn_id, execution.proposal_revision):+ LOG.info(+ "copilot candidate run refused",+ reason="settings_diverged",+ proposal_owner_turn_id=execution.proposal_owner_turn_id,+ proposal_revision=execution.proposal_revision,+ )+ return {"ok": False, "error": COPILOT_PROPOSAL_SETTINGS_CHANGED_ERROR}if explicit_blank and snapshot.provenance == "canonical" and execution.source_is_current(ctx):# Existing verification consumers need the same source this full test selected.@@ -4127,187 +4212,191 @@ async def _run_blocks_and_collect_debug(return _with_build_test_acquisition_context(session_err, requested_block_labels=block_labels)run_session_id = ctx.browser_session_id- execution.recorded_settings = await _browser_execution_settings(workflow, ctx, run_session_id)-- seeded_runtime_workflow = await _workflow_with_runtime_frontier_starter_url_seed(- runtime_workflow,- ctx,- labels_to_execute=labels_to_execute,- runtime_frontier_anchor_url=runtime_frontier_anchor_url,- session_id_override=run_session_id,- )- runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow- runtime_workflow = seeded_runtime_workflow+ dispatch_draft_workflow_id: str | None = None+ # run_task is the in-process inline execution task, only ever set on the dev-only inline path.+ # For dispatched runs it stays None: the worker owns execution and the watchdog observes purely+ # via DB polling.+ run_task: asyncio.Task | None = None+ sensitive_run_custody_lock: asyncio.Lock | None = None+ sensitive_run_session_id: str | None = None+ interim_run_id: str | None = None+ # Set only where the run may already be executing, so an unwind before either executor call+ # can still retract the run-start record.+ run_submission_attempted = False+ # Set before a re-entry of this call, which owns the minted browser from then on.+ keep_run_session = False+ try:+ execution.recorded_settings = await _browser_execution_settings(workflow, ctx, run_session_id)- requested_completion_contract = _requested_completion_contract(ctx, runtime_workflow, labels_to_execute)- if requested_completion_contract is not None:- runtime_workflow = runtime_workflow.model_copy(- update={- "workflow_definition": runtime_workflow.workflow_definition.model_copy(- update={"completion_contract": requested_completion_contract}- )- }+ seeded_runtime_workflow = await _workflow_with_runtime_frontier_starter_url_seed(+ runtime_workflow,+ ctx,+ labels_to_execute=labels_to_execute,+ runtime_frontier_anchor_url=runtime_frontier_anchor_url,+ session_id_override=run_session_id,)+ runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow+ runtime_workflow = seeded_runtime_workflow- # Snapshot version persisted for a worker-dispatched run or an inline run of a definition that- # was never persisted. The run is created against its exact workflow_id so prepare_workflow- # reads parameter rows from the same definition execute_workflow receives. Without the inline- # snapshot, newly drafted parameters exist only in memory and are omitted from the- # WorkflowRunParameter rows, causing block execution to fail before it reaches the browser.- # The snapshot is soft-deleted once the run resolves so it never lingers as the latest version.- dispatch_draft_workflow_id: str | None = None- # The persisted dispatch version (its own regenerated parameter ids) used for post-run output- # mapping on the dispatch path; runtime_workflow / ctx.staged_workflow is left unmutated.- dispatch_workflow: Workflow | None = None- if dispatch_to_worker or snapshot.provenance == "staged" or runtime_workflow != snapshot.workflow:- # Persist the wrapped runtime workflow as a real new version (with its own parameter /- # output-parameter rows) through the normal create machinery. The run is then created- # against this version so the worker resolves it by run.workflow_id and registers block- # outputs from the version's own rows.- try:- dispatch_workflow = await app.WORKFLOW_SERVICE.create_copilot_dispatch_draft_version(- runtime_workflow=runtime_workflow,- organization_id=ctx.organization_id,- )- dispatch_draft_workflow_id = dispatch_workflow.workflow_id- except Exception:- LOG.warning(- "Failed to persist copilot run snapshot; blocking execution",- organization_id=ctx.organization_id,- workflow_permanent_id=ctx.workflow_permanent_id,- dispatch_to_worker=dispatch_to_worker,- snapshot_provenance=snapshot.provenance,- exc_info=True,+ requested_completion_contract = _requested_completion_contract(ctx, runtime_workflow, labels_to_execute)+ if requested_completion_contract is not None:+ runtime_workflow = runtime_workflow.model_copy(+ update={+ "workflow_definition": runtime_workflow.workflow_definition.model_copy(+ update={"completion_contract": requested_completion_contract}+ )+ })- return {- "ok": False,- "error": COPILOT_SNAPSHOT_PREPARATION_ERROR,- "data": {- "workflow_run_id": None,- "failure_reason": COPILOT_SNAPSHOT_PREPARATION_ERROR,- "blocks": [],- },- }- if dispatch_workflow is not None:- snapshot = _materialized_execution_snapshot(snapshot, dispatch_workflow)- # Read before execution: a natural-language loop reference is rewritten on the block while it runs.- execution.loop_definitions = _loop_definitions_or_unavailable(snapshot.workflow)-- if execution.recorded_settings is not None:- # Persistence owns model/header/profile-key settings; the attached browser owns- # the proxy and loaded profile already observed above. None stays unproven.- execution.recorded_settings = copy.deepcopy(- replace(- OriginExecutionSettings.of(snapshot.workflow),- proxy_location=execution.recorded_settings.proxy_location,- browser_profile_id=execution.recorded_settings.browser_profile_id,+ # Snapshot version persisted for a worker-dispatched run or an inline run of a definition that+ # was never persisted. The run is created against its exact workflow_id so prepare_workflow+ # reads parameter rows from the same definition execute_workflow receives. Without the inline+ # snapshot, newly drafted parameters exist only in memory and are omitted from the+ # WorkflowRunParameter rows, causing block execution to fail before it reaches the browser.+ # The snapshot is soft-deleted once the run resolves so it never lingers as the latest version.+ # The persisted dispatch version (its own regenerated parameter ids) used for post-run output+ # mapping on the dispatch path; runtime_workflow / ctx.staged_workflow is left unmutated.+ dispatch_workflow: Workflow | None = None+ if dispatch_to_worker or snapshot.provenance == "staged" or runtime_workflow != snapshot.workflow:+ # Persist the wrapped runtime workflow as a real new version (with its own parameter /+ # output-parameter rows) through the normal create machinery. The run is then created+ # against this version so the worker resolves it by run.workflow_id and registers block+ # outputs from the version's own rows.+ try:+ dispatch_workflow = await app.WORKFLOW_SERVICE.create_copilot_dispatch_draft_version(+ runtime_workflow=runtime_workflow,+ organization_id=ctx.organization_id,+ )+ dispatch_draft_workflow_id = dispatch_workflow.workflow_id+ except Exception:+ LOG.warning(+ "Failed to persist copilot run snapshot; blocking execution",+ organization_id=ctx.organization_id,+ workflow_permanent_id=ctx.workflow_permanent_id,+ dispatch_to_worker=dispatch_to_worker,+ snapshot_provenance=snapshot.provenance,+ exc_info=True,+ )+ return {+ "ok": False,+ "error": COPILOT_SNAPSHOT_PREPARATION_ERROR,+ "data": {+ "workflow_run_id": None,+ "failure_reason": COPILOT_SNAPSHOT_PREPARATION_ERROR,+ "blocks": [],+ },+ }++ if dispatch_workflow is not None:+ snapshot = _materialized_execution_snapshot(snapshot, dispatch_workflow)+ # Read before execution: a natural-language loop reference is rewritten on the block while it runs.+ execution.loop_definitions = _loop_definitions_or_unavailable(snapshot.workflow)++ if execution.recorded_settings is not None:+ # Persistence owns model/header/profile-key settings; the attached browser owns+ # the proxy and loaded profile already observed above. None stays unproven.+ execution.recorded_settings = copy.deepcopy(+ replace(+ OriginExecutionSettings.of(snapshot.workflow),+ proxy_location=execution.recorded_settings.proxy_location,+ browser_profile_id=execution.recorded_settings.browser_profile_id,+ ))- )- execution.dispatched_output_parameter_ids = {- block.label: block.output_parameter.output_parameter_id- for block in snapshot.workflow.workflow_definition.blocks- }+ execution.dispatched_output_parameter_ids = {+ block.label: block.output_parameter.output_parameter_id+ for block in snapshot.workflow.workflow_definition.blocks+ }- all_workflow_params = list(snapshot.workflow_parameters)- all_output_params = list(snapshot.output_parameters)- # The check above resolved every key the dispatched run reads only if persistence added none and dropped none.- unchecked_parameter_keys = (- sorted(origin_checked_parameter_keys ^ {parameter.key for parameter in all_workflow_params})- if origin_checked_resolution is not None- else []- )- refusal = (- selected_output_run_refusal(- ctx,- execution.selected_output_sources,- workflow,- data,- labels_to_execute[0],- execution_settings=execution.recorded_settings,+ all_workflow_params = list(snapshot.workflow_parameters)+ all_output_params = list(snapshot.output_parameters)+ # The check above resolved every key the dispatched run reads only if persistence added none and dropped none.+ unchecked_parameter_keys = (+ sorted(origin_checked_parameter_keys ^ {parameter.key for parameter in all_workflow_params})+ if origin_checked_resolution is not None+ else [])- if execution.selected_output_sources- else None- )- if unchecked_parameter_keys and (execution.selected_output_sources or execution.reused_origin_output_labels):- refusal = logged_origin_refusal(- OriginOutputRefusalDetail(- reason=OriginOutputRefusal.CHANGED_INPUT,- block_label=next(iter(execution.selected_output_sources), None)- or execution.reused_origin_output_labels[0],- origin_workflow_run_id=(- next(iter(execution.selected_output_sources.values())).workflow_run_id- if execution.selected_output_sources- else execution.origin_workflow_run_id- ),- parameter_key=unchecked_parameter_keys[0],+ refusal = (+ selected_output_run_refusal(+ ctx,+ execution.selected_output_sources,+ workflow,+ data,+ labels_to_execute[0],+ execution_settings=execution.recorded_settings,)+ if execution.selected_output_sources+ else None)- if refusal is not None:- # Persistence follows the browser-derived starter URL. Nothing has executed yet:- # retire that preparation and send the original request through security, input- # resolution and browser selection again. With no selected sources this refusal- # cannot trigger another re-entry, even if persistence changes parameter keys again.- if dispatch_draft_workflow_id is not None:- await _delete_dispatch_draft(dispatch_draft_workflow_id, ctx.organization_id)- # Nothing has run in a verified new-exit browser yet, so the re-entry reuses it instead of paying for another.- if used_fresh_run_session and run_session_id and new_exit_receipt is None:- await close_browser_session_quietly(ctx.organization_id, run_session_id)- ctx.frontier_selected_output_sources = {}- ctx.frontier_origin_reused_labels = []- ctx.frontier_resume_session_id = None- ctx.frontier_requires_own_browser = False- requested_labels, requested_seed, requested_start, requested_provenance = _anchored_plan(- ctx, (list(block_labels), {}, block_labels[0]), workflow.workflow_definition, None, list(block_labels)- )- ctx.frontier_start_provenance = requested_provenance- ctx.frontier_origin_output_refusal = refusal- return await _run_blocks_and_collect_debug(- params,- ctx,- labels_to_execute=requested_labels,- block_outputs_to_seed=requested_seed,- frontier_start_label=requested_start,- force_fresh_session=force_fresh_session,- execution_snapshot=execution.snapshot,- explicit_blank=explicit_blank,- use_ephemeral_inputs=use_ephemeral_inputs,- new_exit=new_exit,- verified_new_exit=new_exit_receipt,- )-- execution.dispatched_input_values = copy.deepcopy(data)- ctx.unbound_required_parameter_keys = unbound_required_parameter_keys- execution.reused_origin_input_keys = reused_origin_input_keys- execution.unbound_keys = list(ctx.unbound_required_parameter_keys)- # Only credential-typed values are ever read back; scout-typed form inputs stay out of the record.- execution.parameter_values = {- parameter.key: data[parameter.key]- for parameter in all_workflow_params- if parameter.key in data and parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID- }+ if unchecked_parameter_keys and (execution.selected_output_sources or execution.reused_origin_output_labels):+ refusal = logged_origin_refusal(+ OriginOutputRefusalDetail(+ reason=OriginOutputRefusal.CHANGED_INPUT,+ block_label=next(iter(execution.selected_output_sources), None)+ or execution.reused_origin_output_labels[0],+ origin_workflow_run_id=(+ next(iter(execution.selected_output_sources.values())).workflow_run_id+ if execution.selected_output_sources+ else execution.origin_workflow_run_id+ ),+ parameter_key=unchecked_parameter_keys[0],+ )+ )+ if refusal is not None:+ # Persistence follows the browser-derived starter URL. Nothing has executed yet:+ # retire that preparation and send the original request through security, input+ # resolution and browser selection again. With no selected sources this refusal+ # cannot trigger another re-entry, even if persistence changes parameter keys again.+ if dispatch_draft_workflow_id is not None:+ await _delete_dispatch_draft(dispatch_draft_workflow_id, ctx.organization_id)+ dispatch_draft_workflow_id = None+ # Nothing has run in a verified new-exit browser yet, so the re-entry reuses it instead of paying for another.+ if used_fresh_run_session and run_session_id and new_exit_receipt is None:+ await close_browser_session_quietly(ctx.organization_id, run_session_id)+ keep_run_session = True+ ctx.frontier_selected_output_sources = {}+ ctx.frontier_origin_reused_labels = []+ ctx.frontier_resume_session_id = None+ ctx.frontier_requires_own_browser = False+ requested_labels, requested_seed, requested_start, requested_provenance = _anchored_plan(+ ctx, (list(block_labels), {}, block_labels[0]), workflow.workflow_definition, None, list(block_labels)+ )+ ctx.frontier_start_provenance = requested_provenance+ ctx.frontier_origin_output_refusal = refusal+ return await _run_blocks_and_collect_debug(+ params,+ ctx,+ labels_to_execute=requested_labels,+ block_outputs_to_seed=requested_seed,+ frontier_start_label=requested_start,+ force_fresh_session=force_fresh_session,+ execution_snapshot=execution.snapshot,+ explicit_blank=explicit_blank,+ use_ephemeral_inputs=use_ephemeral_inputs,+ new_exit=new_exit,+ verified_new_exit=new_exit_receipt,+ )- workflow_request = WorkflowRequestBody(- data=data if data else None,- browser_session_id=run_session_id,- # Copilot test runs don't need scrolling post-action screenshots;- # the ForgeAgent's split screenshots (used for LLM context) are unaffected.- max_screenshot_scrolls=0,- )+ execution.dispatched_input_values = copy.deepcopy(data)+ ctx.unbound_required_parameter_keys = unbound_required_parameter_keys+ execution.reused_origin_input_keys = reused_origin_input_keys+ execution.unbound_keys = list(ctx.unbound_required_parameter_keys)+ # Only credential-typed values are ever read back; scout-typed form inputs stay out of the record.+ execution.parameter_values = {+ parameter.key: data[parameter.key]+ for parameter in all_workflow_params+ if parameter.key in data and parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID+ }- # run_task is the in-process inline execution task, only ever set on the dev-only inline path.- # For dispatched runs it stays None: the worker owns execution and the watchdog observes purely- # via DB polling.- run_task: asyncio.Task | None = None- sensitive_run_custody_lock: asyncio.Lock | None = None- sensitive_run_session_id: str | None = None- interim_run_id: str | None = None- current_context = skyvern_context.current()- # Set only where the run may already be executing, so an unwind before either executor call- # can still retract the run-start record.- run_submission_attempted = False- try:+ workflow_request = WorkflowRequestBody(+ data=data if data else None,+ browser_session_id=run_session_id,+ # Copilot test runs don't need scrolling post-action screenshots;+ # the ForgeAgent's split screenshots (used for LLM context) are unaffected.+ max_screenshot_scrolls=0,+ )++ current_context = skyvern_context.current()workflow_run = await workflow_service.prepare_workflow(workflow_id=ctx.workflow_permanent_id,organization=organization,@@ -4355,8 +4444,6 @@ async def _run_blocks_and_collect_debug(if dispatch_draft_workflow_id is not None:await _delete_dispatch_draft(dispatch_draft_workflow_id, ctx.organization_id)dispatch_draft_workflow_id = None- if run_session_id:- await close_browser_session_quietly(ctx.organization_id, run_session_id)return {"ok": False,"error": "The test run's saved browser profile could not be recorded; execution was not started.",@@ -4402,10 +4489,7 @@ async def _run_blocks_and_collect_debug(proposal_owner_turn_id=execution.proposal_owner_turn_id,proposal_revision=execution.proposal_revision,)- return {- "ok": False,- "error": "The pending Copilot proposal changed before the test started; reload and try again.",- }+ return {"ok": False, "error": COPILOT_PROPOSAL_CHANGED_ERROR}ctx.dispatched_run_ids_this_turn.add(workflow_run.workflow_run_id)# A seeded producer's value belongs to the run that produced it, never to this one.seeded_only_labels = frozenset(block_outputs_to_seed) - frozenset(labels_that_may_execute)@@ -4541,6 +4625,11 @@ async def _run_blocks_and_collect_debug(if interim_run_id is not None and not run_submission_attempted:_forget_interim_run_start(ctx, interim_run_id)raise+ finally:+ # Every exit before submission, cancellation included, releases the minted browser; once submission was+ # attempted the run may already hold it.+ if used_fresh_run_session and run_session_id and not run_submission_attempted and not keep_run_session:+ await close_browser_session_quietly(ctx.organization_id, run_session_id)active_run_association: ActiveRunSessionAssociation | None = Nonerun_paused = False@@ -4575,18 +4664,12 @@ async def _run_blocks_and_collect_debug(# completion after the SSE stream is gone so its reply persists# (SKY-8986); aborting mid-block would strand the run without debug# output for the final chat message.- initial_run, initial_step_ts, initial_block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)- progress_marker = _progress_marker(initial_run, initial_step_ts, initial_block_ts)- last_progress_monotonic = time.monotonic()- started_monotonic = last_progress_monotonic+ initial_run, initial_block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)+ started_monotonic = time.monotonic()final_status: str | None = Nonerun: Any = initial_runexit_reason: WatchdogExitReason | None = Nonerun_cancelled_by_watchdog = False- # Quiet blocks (WAIT/TEXT_PROMPT/HUMAN_INTERACTION) legitimately have- # DB-silent periods; disable stagnation for any invocation that includes- # one. Safety ceiling still applies.- stagnation_enabled = not _any_quiet_block_requested(ctx, labels_that_may_execute, workflow=runtime_workflow)budget_seconds = max(1, RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10)narrator_state: NarratorState | None = ctx.narrator_state@@ -4598,7 +4681,7 @@ async def _run_blocks_and_collect_debug(while True:await asyncio.sleep(RUN_BLOCKS_POLL_INTERVAL_SECONDS)- run, step_ts, block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)+ run, block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)if progress_enabled:assert narrator_state is not None@@ -4632,26 +4715,12 @@ async def _run_blocks_and_collect_debug(exit_reason = "task_exit_unfinalized"break- now = time.monotonic()- new_marker = _progress_marker(run, step_ts, block_ts)- # A run in ``paused`` status (e.g. HumanInteractionBlock) is a- # user-driven wait, not stagnation — never trip.- is_paused = run is not None and run.status == WorkflowRunStatus.paused.value- stagnation_active = stagnation_enabled and not is_paused-- if new_marker != progress_marker:- progress_marker = new_marker- last_progress_monotonic = now- elif stagnation_active and now - last_progress_monotonic >= RUN_BLOCKS_STAGNATION_WINDOW_SECONDS:- exit_reason = "stagnation"- break-- if is_paused:+ if run is not None and run.status == WorkflowRunStatus.paused.value:exit_reason = "paused"run_paused = Truebreak- if now - started_monotonic >= budget_seconds:+ if time.monotonic() - started_monotonic >= budget_seconds:exit_reason = "ceiling"break@@ -4664,7 +4733,7 @@ async def _run_blocks_and_collect_debug(# synthetic-``canceled`` ambiguity that the post-drain reread# has to exclude. Then cancel + reread +# ``_trusted_post_drain_status`` applies SKY-9167's success-race- # recovery uniformly to all three non-success exit reasons.+ # recovery uniformly to both non-success exit reasons.pre_cancel_run = await _safe_read_workflow_run(workflow_run.workflow_run_id, ctx.organization_id, context="pre-cancel")@@ -5230,6 +5299,14 @@ async def _get_run_results(run_workflow = Noneif run_workflow is None:LOG.warning("Prior-run workflow snapshot fetch failed", workflow_run_id=workflow_run_id)+ # Only a row proven to be what the run executed may describe it; an unproven row is treated as missing,+ # which every consumer below already handles fail-closed (including sensitivity).+ executed_workflow = (+ run_workflow+ if workflow_run_id == ctx.proposal_workflow_run_id+ or _run_version_unproven_reason(ctx, run, run_workflow) is None+ else None+ )origin_registry = getattr(ctx, "origin_run_redaction_registry", None)matching_origin_registry = (@@ -5237,7 +5314,7 @@ async def _get_run_results()# Unknown provenance is not evidence that a cold run was nonsensitive. Use the same# fail-closed classification for every direct page producer and for terminal artifacts.- workflow_has_sensitive_parameters = _workflow_requires_terminal_artifact_redaction(run_workflow)+ workflow_has_sensitive_parameters = _workflow_requires_terminal_artifact_redaction(executed_workflow)sensitive_origin_run = (_origin_registry_contains_sensitive_values(matching_origin_registry, workflow_run_id)or workflow_has_sensitive_parameters@@ -5287,9 +5364,19 @@ async def _get_run_results("action_trace_summary": action_trace_summary,"action_observations": action_observations,}- if run_workflow is not None and workflow_run_id == ctx.proposal_workflow_run_id:- result_data["execution_source"] = _execution_source_facts(run_workflow, provenance="staged")- loop_inputs, loop_input_notices = _read_run_loop_inputs(run_workflow, run.created_at, workflow_run_id, blocks)+ if workflow_run_id == ctx.proposal_workflow_run_id:+ if run_workflow is not None:+ result_data["execution_source"] = _execution_source_facts(run_workflow, provenance="staged")+ else:+ result_data["execution_source"] = _run_version_source(+ ctx,+ run,+ run_workflow,+ failed_block_label=_packet_string(newest_failed.get("label")) if newest_failed else None,+ failing_line=result_data["failing_code_line"],+ requested_block_labels=result_data["requested_block_labels"],+ )+ loop_inputs, loop_input_notices = _read_run_loop_inputs(executed_workflow, workflow_run_id, blocks)if loop_inputs is not None:result_data[LOOP_INPUTS_KEY] = loop_inputs.model_dump(mode="json", exclude_none=True)if loop_input_notices:@@ -5314,7 +5401,9 @@ async def _get_run_results(and not sensitive_origin_runand not _run_browser_carries_a_sign_in(ctx, run.browser_session_id)):- failed_block_code = _failed_block_code(run_workflow, newest_failed) if run_workflow is not None else None+ failed_block_code = (+ _failed_block_code(executed_workflow, newest_failed) if executed_workflow is not None else None+ )locator_observations = await _observe_authored_locators(ctx,run_session_id=run.browser_session_id,@@ -5351,26 +5440,30 @@ async def _get_run_results(workflow_run_id=workflow_run_id,)- if run_workflow is None:+ if executed_workflow is None:result_data["requested_output_definitions_omission"] = "the run-pinned workflow snapshot was unavailable"else:result_data["requested_output_parameter_definitions"] = _requested_output_parameter_definitions(workflow_run_id=workflow_run_id,- workflow=run_workflow,+ workflow=executed_workflow,)await _attach_registered_output_parameter_values(workflow_run_id=workflow_run_id,- workflow=run_workflow,+ workflow=executed_workflow,data=result_data,persisted_output_parameters=(- [parameter for parameter in _workflow_parameters(run_workflow) if isinstance(parameter, OutputParameter)]- if run_workflow is not None+ [+ parameter+ for parameter in _workflow_parameters(executed_workflow)+ if isinstance(parameter, OutputParameter)+ ]+ if executed_workflow is not Noneelse None),excluded_block_labels=ctx.seeded_only_labels_by_run_id.get(workflow_run_id, frozenset()),)- cold_artifact_requires_redaction_context = _workflow_requires_terminal_artifact_redaction(run_workflow)+ cold_artifact_requires_redaction_context = _workflow_requires_terminal_artifact_redaction(executed_workflow)# Cold ordinary-turn hydration did not perform a sensitive run and cannot use mutable context# alone to vouch for its raw artifact custody. The direct same-turn tool route may use its exact# immutable registry; cold sensitive hydration receives a typed omission instead.@@ -5389,7 +5482,7 @@ async def _get_run_results(run_id=workflow_run_id,organization_id=ctx.organization_id,current_url=current_url,- workflow=run_workflow,+ workflow=executed_workflow,origin_redaction_registry=artifact_redaction_registry,))@@ -6411,7 +6504,14 @@ def _build_recorded_build_test_outcome(workflow_definition = getattr(copilot_ctx.last_workflow, "workflow_definition", None)raw_requested_block_labels = getattr(copilot_ctx, "last_requested_block_labels", None)execution = result.execution if isinstance(result, _ExecutionResult) else None- if execution is not None:+ historical_run = _is_run_version_result(result_data)+ if historical_run:+ # The chat's draft is not what a finished run executed, so none of its signatures describe this run.+ workflow_yaml = None+ code_artifact_metadata = {}+ workflow_definition = None+ raw_requested_block_labels = result_data.get("requested_block_labels")+ elif execution is not None:workflow_yaml = execution.workflow_yamlcode_artifact_metadata = execution.metadataworkflow_definition = execution.snapshot.workflow.workflow_definition@@ -6449,7 +6549,7 @@ def _build_recorded_build_test_outcome(declared_goal_path_omissions=declared_goal_path_omissions,unbound_required_parameter_keys=execution.unbound_keysif execution- else list(copilot_ctx.unbound_required_parameter_keys),+ else ([] if historical_run else list(copilot_ctx.unbound_required_parameter_keys)),block_parameter_keys=authored_block_parameter_keys_from_workflow(workflow_yaml,code_artifact_metadata,@@ -6457,7 +6557,7 @@ def _build_recorded_build_test_outcome(block_shape_hashes=block_shape_hashes,block_associations_by_label=execution.associationsif execution- else getattr(copilot_ctx, "runner_code_block_associations_by_label", {}),+ else ({} if historical_run else getattr(copilot_ctx, "runner_code_block_associations_by_label", {})),)raw_blocks = result_data.get("blocks")ai_fallback_blocks = _packet_ai_fallback_blocks(raw_blocks if isinstance(raw_blocks, list) else [])@@ -7249,20 +7349,21 @@ def _packet_unfinished_items(recorded_outcome: RecordedBuildTestOutcome | None,declared_goal_path_omissions: object,omission_notices: list[str],+ unverified_block_labels: Sequence[str],) -> list[BuildTestPacketUnfinishedItem]:latest_outcome = getattr(copilot_ctx, "latest_recorded_build_test_outcome", None)outcome = recorded_outcome or (latest_outcome if isinstance(latest_outcome, RecordedBuildTestOutcome) else None)if run_id is None:- if copilot_ctx.last_unverified_block_labels or outcome is not None:+ if unverified_block_labels or outcome is not None:omission_notices.append("unfinished_items omitted: recorded unfinished evidence is not bound to this run.")return []outcome_matches_run = outcome is not None and outcome.workflow_run_id == run_id- if not outcome_matches_run and (copilot_ctx.last_unverified_block_labels or outcome is not None):+ if not outcome_matches_run and (unverified_block_labels or outcome is not None):omission_notices.append("unfinished_items omitted: recorded unfinished evidence is not bound to this run.")unfinished: list[BuildTestPacketUnfinishedItem] = ([BuildTestPacketUnfinishedItem(kind="unverified_block", label=label)- for label in dict.fromkeys(copilot_ctx.last_unverified_block_labels)+ for label in dict.fromkeys(unverified_block_labels)if isinstance(label, str) and label]if outcome_matches_run@@ -7472,6 +7573,8 @@ def build_test_evidence_packet(recorded_outcome,raw_goal_path_omissions,omission_notices,+ # The chat's unverified labels describe its current draft, not the version a finished run executed.+ unverified_block_labels=[] if _is_run_version_result(data) else copilot_ctx.last_unverified_block_labels,)if not unfinished_items:omission_notices.append("unfinished_items empty: recorded outcome and workflow evidence identify none.")diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 25245cd05..66b4cddf7 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -16,7 +16,7 @@ from typing import Annotated, Any, Literal, NamedTuple, castimport structlogimport yamlfrom jinja2 import TemplateSyntaxError-from pydantic import AliasChoices, BaseModel, Field, ValidationError+from pydantic import AliasChoices, BaseModel, Field, TypeAdapter, ValidationErrorfrom skyvern.constants import DEFAULT_WORKFLOW_TITLESfrom skyvern.exceptions import SkyvernHTTPException@@ -114,6 +114,8 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (workflow_blocks,)from skyvern.forge.sdk.copilot.workflow_yaml import (+ _PRIVATE_WORKFLOW_SETTINGS_FIELDS,+ _canonicalize_copilot_proxy_location,_normalize_copilot_yaml,_process_workflow_yaml,dump_workflow_yaml,@@ -140,7 +142,7 @@ from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, Insfrom skyvern.forge.sdk.workflow.models.block import CodeBlockfrom skyvern.forge.sdk.workflow.models.workflow import Workflowfrom skyvern.forge.sdk.workflow.runtime_completion import contract_from_code_artifact_metadata-from skyvern.schemas.proxy_location import runtime_proxy_location+from skyvern.schemas.proxy_location import ProxyLocationInput, runtime_proxy_locationfrom skyvern.schemas.workflows import (BLOCK_YAML_SUBCLASSES,BlockType,@@ -3990,7 +3992,9 @@ def strip_copilot_yaml_headers(workflow_yaml: str | None) -> str | None:return None-def copilot_workflow_yaml_matches(submitted_yaml: str | None, *known_sources: str | None) -> bool:+def copilot_workflow_yaml_matches(+ submitted_yaml: str | None, *known_sources: str | None, inherited_proxy_location: ProxyLocationInput = None+) -> bool:submitted_yaml = strip_copilot_yaml_headers(submitted_yaml)if not submitted_yaml:return False@@ -3999,10 +4003,14 @@ def copilot_workflow_yaml_matches(submitted_yaml: str | None, *known_sources: stsanitized_source = strip_copilot_yaml_headers(source)if not sanitized_source:continue- source_canvas = _normalized_canvas_for_proposal_restore(sanitized_source)+ source_canvas = _normalized_canvas_for_proposal_restore(+ sanitized_source, inherited_proxy_location=inherited_proxy_location+ )if (_normalized_canvas_for_proposal_restore(- submitted_yaml, inherited_code_version=source_canvas.code_version+ submitted_yaml,+ inherited_code_version=source_canvas.code_version,+ inherited_proxy_location=inherited_proxy_location,)== source_canvas):@@ -4016,6 +4024,7 @@ def _normalized_canvas_for_proposal_restore(workflow_yaml: str,*,inherited_code_version: int | None = None,+ inherited_proxy_location: ProxyLocationInput = None,) -> WorkflowCreateYAMLRequest:"""Normalize editor-only empty code outlines before comparing canvas custody.@@ -4025,6 +4034,7 @@ def _normalized_canvas_for_proposal_restore("""normalized = _normalize_copilot_yaml(workflow_yaml)normalized.webhook_callback_url = normalized.webhook_callback_url or None+ normalized.proxy_location = resolve_proxy_location(normalized, inherited_proxy_location)normalized.extra_http_headers = Nonenormalized.cdp_connect_headers = Nonenormalized.mask_secrets = bool(normalized.mask_secrets)@@ -4113,6 +4123,7 @@ async def _validated_pending_workflow_proposal(proposal_workflow_run_id=metadata.workflow_run_id if metadata is not None else None,)return None+ canonical: Workflow | None = Noneif metadata is not None:canonical = await app.DATABASE.workflows.get_workflow_by_permanent_id(workflow_permanent_id=ctx.workflow_permanent_id,@@ -4138,7 +4149,10 @@ async def _validated_pending_workflow_proposal(# An explicit canvas edit remains the model's input to the normal update tool.# Only restore over the persisted canvas or the same pending proposal.if ctx.workflow_yaml and not copilot_workflow_yaml_matches(- ctx.workflow_yaml, sanitized_yaml, ctx.persisted_workflow_yaml+ ctx.workflow_yaml,+ sanitized_yaml,+ ctx.persisted_workflow_yaml,+ inherited_proxy_location=canonical.proxy_location if canonical is not None else None,):LOG.info("copilot_pending_proposal_restore_rejected",@@ -4149,6 +4163,42 @@ async def _validated_pending_workflow_proposal(return proposal, sanitized_yaml+_PROXY_LOCATION_ADAPTER = TypeAdapter(WorkflowCreateYAMLRequest.model_fields["proxy_location"].annotation)+++def _effective_private_setting(name: str, value: object) -> object:+ # Accept reads a null header map as {} and an empty string as unset, and validates proxy spellings the way+ # canvas custody does.+ if name in ("extra_http_headers", "cdp_connect_headers"):+ return value or {}+ if name != "proxy_location":+ return value or None+ if value is None:+ return None+ canonical = {"proxy_location": value}+ _canonicalize_copilot_proxy_location(canonical)+ try:+ return _PROXY_LOCATION_ADAPTER.validate_python(canonical["proxy_location"])+ except ValidationError:+ return value+++def _written_private_setting(name: str, value: object) -> object:+ # Equal effective headers can still differ in written keys: an unbound mask is an explicit entry Accept resolves.+ value = _effective_private_setting(name, value)+ if isinstance(value, dict) and name in ("extra_http_headers", "cdp_connect_headers"):+ return dict.fromkeys(value, SECRET_HEADER_MASK)+ return value+++def _effective_private_settings(*sources: Mapping[str, object]) -> dict[str, object]:+ # The first source that names a setting wins; Accept fills settings the proposal omits from the saved workflow.+ return {+ name: _effective_private_setting(name, next((source[name] for source in sources if name in source), None))+ for name in _PRIVATE_WORKFLOW_SETTINGS_FIELDS+ }++async def restore_pending_workflow_proposal(ctx: CopilotContext,*,@@ -4182,21 +4232,18 @@ async def restore_pending_workflow_proposal(saved = await app.DATABASE.workflows.get_workflow_by_permanent_id(workflow_permanent_id=ctx.workflow_permanent_id, organization_id=ctx.organization_id)+ inherited_settings = inherited_header_settings(saved) if saved is not None else Noneprivate_settings = merge_private_workflow_settings(- proposal_settings,- request_settings,- inherited_settings=inherited_header_settings(saved) if saved is not None else None,+ proposal_settings, request_settings, inherited_settings=inherited_settings)ctx.private_workflow_settings = private_settingsif pending is None:return_, sanitized_yaml = pendingmetadata = copilot_proposal_metadata(proposal)- restored_yaml = sanitized_yaml- if private_settings:- restored_document = safe_load_no_dates(sanitized_yaml)- restored_document.update(private_settings)- restored_yaml = dump_workflow_yaml(restored_document)+ stored_document = safe_load_no_dates(sanitized_yaml)+ restored_document = {**stored_document, **private_settings}+ restored_yaml = dump_workflow_yaml(restored_document) if private_settings else sanitized_yamlif "workflow_definition" in proposal:workflow = Workflow.model_validate(proposal)# A proposal froze its title in an earlier turn, so a rename saved between the two outranks@@ -4238,7 +4285,39 @@ async def restore_pending_workflow_proposal(settings_fallback_yaml=ctx.persisted_workflow_yaml,prefer_live_title=True,)- workflow_yaml = strip_copilot_yaml_headers(restored_yaml)+ # Serialization masks CDP headers, so the saved side takes its real header maps from the row itself.+ saved_settings = (+ {**submitted_private_workflow_settings(saved.model_dump(mode="json")), **inherited_header_settings(saved)}+ if saved is not None+ else {}+ )+ own_settings = merge_private_workflow_settings(proposal_settings, inherited_settings=inherited_settings)+ own_effective = _effective_private_settings(own_settings, stored_document, saved_settings)+ effective = _effective_private_settings(private_settings, stored_document, saved_settings)+ if effective == own_effective:+ # The turn-end publish compares settings as written, so a request value that only echoes the proposal's+ # effective value is recorded as the proposal wrote it.+ for name, value in request_settings.items():+ if _written_private_setting(name, value) != _written_private_setting(name, own_effective.get(name)):+ continue+ if name in proposal_settings:+ ctx.authored_private_workflow_settings[name] = copy.deepcopy(proposal_settings[name])+ else:+ ctx.authored_private_workflow_settings.pop(name, None)+ elif metadata is not None:+ ctx.settings_diverged_proposal_token = (metadata.owner_turn_id, metadata.revision)+ LOG.info(+ "copilot_pending_proposal_settings_diverged",+ workflow_permanent_id=ctx.workflow_permanent_id,+ proposal_owner_turn_id=metadata.owner_turn_id,+ proposal_revision=metadata.revision,+ setting_names=sorted(name for name in own_effective if effective[name] != own_effective[name]),+ )+ strip_private_workflow_settings(own_effective)+ strip_private_workflow_settings(effective)+ # Run bind compares staged bytes to stored ones, and the model's next edit re-saves what it sees, so re-dump+ # only when a visible setting changed.+ workflow_yaml = sanitized_yaml if own_effective == effective else strip_copilot_yaml_headers(restored_yaml)ctx.staged_workflow = workflowctx.staged_workflow_yaml = workflow_yamlctx.workflow_yaml = workflow_yamldiff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 636ab01a5..593b17742 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -952,6 +952,7 @@ class WorkflowRunModel(Base):start_fresh_browser = Column(Boolean, nullable=True)reuse_browser_session = Column(Boolean, nullable=True)reuse_bound_key = Column(String, nullable=True)+ workflow_definition_sha256 = Column(String, nullable=True)status = Column(String, nullable=False)failure_reason = Column(String)proxy_location = Column(String)@@ -1928,6 +1929,15 @@ class CredentialModel(Base):class DebugSessionModel(Base):__tablename__ = "debug_sessions"+ __table_args__ = (+ Index(+ "ix_debug_sessions_org_wpid_user_created_at",+ "organization_id",+ "workflow_permanent_id",+ "user_id",+ "created_at",+ ),+ )debug_session_id = Column(String, primary_key=True, default=generate_debug_session_id)organization_id = Column(String, nullable=False)diff --git a/skyvern/forge/sdk/db/repositories/debug.py b/skyvern/forge/sdk/db/repositories/debug.pyindex ed59819a7..5f51f4c12 100644--- a/skyvern/forge/sdk/db/repositories/debug.py+++ b/skyvern/forge/sdk/db/repositories/debug.py@@ -34,6 +34,7 @@ class DebugRepository(BaseRepository):.filter_by(deleted_at=None).filter_by(status="created").order_by(DebugSessionModel.created_at.desc())+ .limit(1))).first()diff --git a/skyvern/forge/sdk/db/repositories/tasks.py b/skyvern/forge/sdk/db/repositories/tasks.pyindex 671f11209..8879af7dc 100644--- a/skyvern/forge/sdk/db/repositories/tasks.py+++ b/skyvern/forge/sdk/db/repositories/tasks.py@@ -265,38 +265,19 @@ class TasksRepository(BaseRepository):total = (await session.execute(query)).scalar_one()return float(total)- @db_operation("get_workflow_run_progress_timestamps")- async def get_workflow_run_progress_timestamps(+ @db_operation("get_workflow_run_block_progress_timestamp")+ async def get_workflow_run_block_progress_timestamp(self,workflow_run_id: str,organization_id: str | None = None,- ) -> tuple[datetime | None, datetime | None]:- """Return ``(max(step.modified_at), max(workflow_run_block.modified_at))``- for a given workflow run. Both values are scalar aggregates — no row- hydration — and are designed for the copilot watchdog poll path where- the only question is "has anything changed since the last poll?".-- Step updates are the per-LLM-call heartbeat (status transitions plus- incremental token/cost accumulators — every ``update_step`` call bumps- ``StepModel.modified_at``). Block updates cover non-task block types- (CODE, TEXT_PROMPT) that never create a task row. Together the two- aggregates cover every kind of block the copilot can run.- """+ ) -> datetime | None:async with self.Session() as session:- step_stmt = (- select(func.max(StepModel.modified_at))- .join(TaskModel, StepModel.task_id == TaskModel.task_id)- .where(TaskModel.workflow_run_id == workflow_run_id)- .where(StepModel.organization_id == organization_id)- )block_stmt = (select(func.max(WorkflowRunBlockModel.modified_at)).where(WorkflowRunBlockModel.workflow_run_id == workflow_run_id).where(WorkflowRunBlockModel.organization_id == organization_id))- step_ts = (await session.execute(step_stmt)).scalar_one_or_none()- block_ts = (await session.execute(block_stmt)).scalar_one_or_none()- return step_ts, block_ts+ return (await session.execute(block_stmt)).scalar_one_or_none()@db_operation("get_total_unique_step_order_count_by_task_ids")async def get_total_unique_step_order_count_by_task_ids(diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.pyindex f4053355e..76012af94 100644--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py@@ -92,6 +92,7 @@ from skyvern.forge.sdk.workflow.constants import INTERIM_OUTPUT_SNAPSHOT_MAX_BYTfrom skyvern.forge.sdk.workflow.credential_selection import clear_credential_selections_for_retryfrom skyvern.forge.sdk.workflow.models.parameter import WorkflowParameterfrom skyvern.forge.sdk.workflow.models.workflow import (+ MIXED_RUN_DEFINITION_DIGEST,WorkflowDefinition,WorkflowRun,WorkflowRunOutputParameter,@@ -863,6 +864,7 @@ class WorkflowRunsRepository(BaseRepository):copilot_session_id: str | None = None,start_fresh_browser: bool | None = None,created_by: str | None = None,+ workflow_definition_sha256: str | None = None,) -> WorkflowRun:async with self.Session() as session:kwargs: dict[str, Any] = {}@@ -877,6 +879,7 @@ class WorkflowRunsRepository(BaseRepository):start_fresh_browser=start_fresh_browser,reuse_browser_session=reuse_browser_session,reuse_bound_key=reuse_bound_key,+ workflow_definition_sha256=workflow_definition_sha256,proxy_location=serialize_proxy_location(proxy_location),status="created",webhook_callback_url=webhook_callback_url,@@ -1220,6 +1223,7 @@ class WorkflowRunsRepository(BaseRepository):only_from: Sequence[WorkflowRunStatus] | None = None,job_id: str | None = None,depends_on_workflow_run_id: str | None = None,+ workflow_definition_sha256: str | None = None,) -> WorkflowRun | None:"""One conditional UPDATE to ``status`` from a non-terminal state (``only_from`` narrows it), so a latecancel cannot clobber a finalization; None when the row was terminal or missing. Timestamps follow@@ -1245,6 +1249,14 @@ class WorkflowRunsRepository(BaseRepository):values["job_id"] = job_idif depends_on_workflow_run_id:values["depends_on_workflow_run_id"] = depends_on_workflow_run_id+ if workflow_definition_sha256 is not None:+ # Only invalidate: a retry re-enters with the row as it is now while earlier attempts' block rows still+ # describe the old code, and a NULL digest (a run created before digests) cannot say what those ran.+ stored = WorkflowRunModel.workflow_definition_sha256+ values["workflow_definition_sha256"] = case(+ (and_(stored.is_not(None), stored != workflow_definition_sha256), MIXED_RUN_DEFINITION_DIGEST),+ else_=stored,+ )# The reopen/reset path clears attribution when it returns the row to `created`, so a# later terminal transition starts from SQL NULL and this COALESCE fills the freshly# derived document; on any row that already carries one it preserves the first writer.diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex 89311a53e..1acc826c9 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -700,6 +700,7 @@ def convert_to_workflow_run(start_fresh_browser=workflow_run_model.start_fresh_browser,reuse_browser_session=workflow_run_model.reuse_browser_session,reuse_bound_key=workflow_run_model.reuse_bound_key,+ workflow_definition_sha256=workflow_run_model.workflow_definition_sha256,task_queue=workflow_run_model.task_queue,status=WorkflowRunStatus[workflow_run_model.status],failure_reason=workflow_run_model.failure_reason,diff --git a/skyvern/forge/sdk/routes/credentials.py b/skyvern/forge/sdk/routes/credentials.pyindex 8c5069ce3..8450566ce 100644--- a/skyvern/forge/sdk/routes/credentials.py+++ b/skyvern/forge/sdk/routes/credentials.py@@ -704,14 +704,23 @@ async def create_credential(current_org: Organization = Depends(org_auth_service.get_current_org_for_credential_routes),current_user_id: str | None = Depends(org_auth_service.get_current_user_id_or_none),) -> CredentialResponse:+ credential_service = await prepare_credential_create(current_org.organization_id, data)+ credential = await store_new_credential(+ current_org.organization_id, data, current_user_id, credential_service, background_tasks+ )+ return _convert_to_response(credential)+++async def prepare_credential_create(organization_id: str, data: CreateCredentialRequest) -> CredentialVaultService:+ """Every check a create runs before it touches a vault; raising here means nothing was stored."""if isinstance(data.credential, NonEmptyPasswordCredential):await _normalize_authenticator_totp_for_organization_or_raise(data.credential,- organization_id=current_org.organization_id,+ organization_id=organization_id,)await app.AGENT_FUNCTION.validate_credential_write(- organization_id=current_org.organization_id,+ organization_id=organization_id,credential_type=data.credential_type,credential=data.credential,existing_credential=None,@@ -720,18 +729,26 @@ async def create_credential(# Validate the profile link BEFORE provisioning, so an invalid/managed/cross-org/owned profile# rejects with a 400 without leaving an orphan credential (and vault secret) behind.if data.browser_profile_id is not None:- await _validate_credential_browser_profile_id(data.browser_profile_id, current_org.organization_id)+ await _validate_credential_browser_profile_id(data.browser_profile_id, organization_id)- credential_service = await _get_credential_vault_service(+ return await _get_credential_vault_service(vault_type_override=data.vault_type,- organization_id=current_org.organization_id,+ organization_id=organization_id,)++async def store_new_credential(+ organization_id: str,+ data: CreateCredentialRequest,+ created_by: str | None,+ credential_service: CredentialVaultService,+ background_tasks: BackgroundTasks,+) -> Credential:try:credential = await credential_service.create_credential(- organization_id=current_org.organization_id,+ organization_id=organization_id,data=data,- created_by=current_user_id,+ created_by=created_by,)except SkyvernHttpException as e:detail = (@@ -739,11 +756,9 @@ async def create_credential(if e.error_messageelse f"Custom credential service returned HTTP {e.status_code}")- raise HTTPException(status_code=502, detail=detail)+ raise HTTPException(status_code=502, detail=detail) from e- await record_request_audit_event(- current_org.organization_id, "credential.create", "credential", credential.credential_id- )+ await record_request_audit_event(organization_id, "credential.create", "credential", credential.credential_id)if credential.vault_type == CredentialVaultType.BITWARDEN:# Early resyncing the Bitwarden vaultbackground_tasks.add_task(fetch_credential_item_background, credential.item_id)@@ -757,7 +772,7 @@ async def create_credential(profile_update: dict[str, Any] = {"browser_profile_id": data.browser_profile_id} if bpid_provided else {}credential = await _update_credential_or_profile_conflict(credential_id=credential.credential_id,- organization_id=current_org.organization_id,+ organization_id=organization_id,**profile_update,# Only touch the pin when the caller sent it — supplying a profile alone must not reset it.pin_saved_session_ip=data.pin_saved_session_ip if pin_provided else None,@@ -766,14 +781,13 @@ async def create_credential(background_tasks.add_task(app.AGENT_FUNCTION.on_credential_saved,- organization_id=current_org.organization_id,+ organization_id=organization_id,credential_id=credential.credential_id,credential_type=data.credential_type,- actor_user_id=current_user_id,+ actor_user_id=created_by,vault=credential.vault_type.value if credential.vault_type is not None else None,)-- return _convert_to_response(credential)+ return credentialLOGIN_TEST_PROMPT = (diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex f9faa3dd2..b9fd3ab47 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -15,7 +15,7 @@ from urllib.parse import urlparseimport structlogimport yaml-from fastapi import Depends, File, Form, Header, HTTPException, Request, UploadFile, status+from fastapi import BackgroundTasks, Depends, File, Form, Header, HTTPException, Request, UploadFile, statusfrom opentelemetry import trace as otel_tracefrom pydantic import ValidationErrorfrom sse_starlette import EventSourceResponse@@ -23,6 +23,7 @@ from sse_starlette import EventSourceResponsefrom skyvern import analyticsfrom skyvern.config import settingsfrom skyvern.constants import DEFAULT_WORKFLOW_TITLES+from skyvern.exceptions import HttpException as VaultHttpExceptionfrom skyvern.exceptions import SkyvernHTTPExceptionfrom skyvern.forge import appfrom skyvern.forge.agent_functions import AuditEvent@@ -47,13 +48,17 @@ from skyvern.forge.sdk.copilot.context import (clear_proposed_credential,merge_approved_credentials_into_global_llm_context,)+from skyvern.forge.sdk.copilot.credential_generation import generate_registration_passwordfrom skyvern.forge.sdk.copilot.credential_pause import (+ MANUAL_SIGN_IN_CLAIM_SECONDS,MANUAL_SIGN_IN_SAVE_TIMEOUT_SECONDS,CredentialPauseRejection,SignedInProfile,check_credential_pause_resumable,+ claim_credential_generation,claim_manual_sign_in,credential_pause_is_active,+ finish_credential_generation,finish_manual_sign_in,pending_credential_requests,resolve_credential_pause,@@ -119,8 +124,16 @@ from skyvern.forge.sdk.routes.browser_profiles import (_hard_delete_created_profile_after_store_failure,create_profile_from_running_session,)+from skyvern.forge.sdk.routes.credentials import prepare_credential_create, store_new_credentialfrom skyvern.forge.sdk.routes.routers import base_routerfrom skyvern.forge.sdk.schemas.copilot_turn_outcome import PersistedCopilotComposerMode, ResponseKind, TurnOutcome+from skyvern.forge.sdk.schemas.credentials import (+ CreateCredentialRequest,+ Credential,+ CredentialType,+ NonEmptyPasswordCredential,+ PasswordCredential,+)from skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.workflow_copilot import (COPILOT_PRIVATE_SETTINGS_KEY,@@ -145,6 +158,8 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatSender,WorkflowCopilotChatSummary,WorkflowCopilotClearProposedWorkflowRequest,+ WorkflowCopilotCredentialGenerateRequest,+ WorkflowCopilotCredentialGenerateResult,WorkflowCopilotCredentialResponseRequest,WorkflowCopilotCredentialResponseResult,WorkflowCopilotDisableAutoAcceptRequest,@@ -163,6 +178,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (copilot_proposal_metadata,)from skyvern.forge.sdk.services import org_auth_service+from skyvern.forge.sdk.services.credential.credential_vault_service import CredentialVaultServicefrom skyvern.forge.sdk.services.request_principal import get_request_principalfrom skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPExceptionfrom skyvern.forge.sdk.workflow.models.workflow import Workflow@@ -1044,9 +1060,20 @@ async def _persist_proposed_workflow_state(proposed_workflow_data = _build_proposed_workflow_data(updated_workflow, agent_result)if agent_result.proposal_owner_turn_id is not None and agent_result.proposal_revision is not None:stored_metadata = copilot_proposal_metadata(chat.proposed_workflow)+ stored_yaml = (+ chat.proposed_workflow.get("_copilot_yaml") if isinstance(chat.proposed_workflow, dict) else None+ )same_bytes = (isinstance(chat.proposed_workflow, dict)- and chat.proposed_workflow.get("_copilot_yaml") == proposed_workflow_data.get("_copilot_yaml")+ and (+ stored_yaml == proposed_workflow_data.get("_copilot_yaml")+ # A rename saved since publication re-titles a turn that answered with the stored bytes;+ # Accept resolves the rename.+ or (+ stored_yaml is not None+ and stored_yaml == strip_copilot_yaml_headers(agent_result.workflow_yaml)+ )+ )and private_workflow_settings_from_proposal(chat.proposed_workflow)== private_workflow_settings_from_proposal(proposed_workflow_data))@@ -1055,7 +1082,11 @@ async def _persist_proposed_workflow_state(# markers. Those bytes carry the resolved title and Accept reparses them as they are.proposed_workflow_data = {**chat.proposed_workflow,- **{key: value for key, value in proposed_workflow_data.items() if key.startswith("_copilot_")},+ **{+ key: value+ for key, value in proposed_workflow_data.items()+ if key.startswith("_copilot_") and key != "_copilot_yaml"+ },}if _proposal_disposition(agent_result) != "review_untested":# The marker is only ever added, so a candidate published untested and since@@ -2667,6 +2698,8 @@ async def _new_copilot_chat_post(detail="No pending proposal to test end to end.",)_apply_test_end_to_end_action(chat_request, validated_proposal["_copilot_yaml"])+ # The Test action runs the proposal under its own settings, never the editor's.+ submitted_private_settings = {}await stream.send(WorkflowCopilotProcessingUpdate(@@ -4256,6 +4289,190 @@ async def workflow_copilot_credential_response(return WorkflowCopilotCredentialResponseResult(result="accepted")+_READBACK_PAGE_SIZE = 100+++async def _find_generated_credential(+ organization_id: str,+ data: CreateCredentialRequest,+ created_by: str | None,+ created_since: datetime,+ credential_service: CredentialVaultService,+) -> str | None:+ """Return the one credential this card saved, or None when no single row holds the password it generated."""+ credentials = await app.DATABASE.credentials.get_credentials(+ organization_id, page_size=_READBACK_PAGE_SIZE, credential_type=CredentialType.PASSWORD, search=data.name+ )++ def created_in_window(credential: Credential) -> bool:+ return credential.created_at.replace(tzinfo=credential.created_at.tzinfo or UTC) >= created_since++ # Rows come newest first, so a full page whose oldest row is still in the window may hide more matches.+ if len(credentials) >= _READBACK_PAGE_SIZE and created_in_window(credentials[-1]):+ return None+ matches = [+ credential+ for credential in credentials+ if credential.name == data.name+ and credential.username == data.credential.username+ and credential.created_by == created_by+ and created_in_window(credential)+ ]+ if len(matches) != 1:+ return None+ stored = (await credential_service.get_credential_item(matches[0])).credential+ # Matching metadata alone could name another create of the same account, so only the generated password counts.+ if isinstance(stored, PasswordCredential) and hmac.compare_digest(+ stored.password.encode(), data.credential.password.encode()+ ):+ return matches[0].credential_id+ return None+++def _log_late_create(stored: asyncio.Future[Credential]) -> None:+ if not stored.cancelled() and stored.exception() is None:+ LOG.warning("copilot_credential_generation_late_create", credential_id=stored.result().credential_id)+++_DETACHED_CREDENTIAL_CREATES: set[asyncio.Task[None]] = set()+++async def _create_generated_credential(+ organization_id: str,+ data: CreateCredentialRequest,+ created_by: str | None,+ credential_service: CredentialVaultService,+ stored: asyncio.Future[Credential],+) -> None:+ # The request's own BackgroundTasks never run once it has answered, so a late save brings its own.+ hooks = BackgroundTasks()+ try:+ credential = await store_new_credential(organization_id, data, created_by, credential_service, hooks)+ except Exception as exc:+ stored.set_exception(exc)+ return+ except asyncio.CancelledError:+ stored.cancel()+ raise+ stored.set_result(credential)+ # Nothing awaits this task, so a failing post-save hook would otherwise vanish as an unretrieved exception.+ try:+ await hooks()+ except Exception:+ LOG.warning("copilot_credential_generation_post_save_hooks_failed", exc_info=True)+++# Every wait leaves the claim the same margin to finish in that the manual sign-in save does.+_CLAIM_FINISH_MARGIN_SECONDS = MANUAL_SIGN_IN_CLAIM_SECONDS - MANUAL_SIGN_IN_SAVE_TIMEOUT_SECONDS+_READBACK_RESERVE_SECONDS = 15+++def _claim_seconds_left(deadline: datetime) -> float:+ return max(0.0, (deadline - datetime.now(UTC)).total_seconds() - _CLAIM_FINISH_MARGIN_SECONDS)+++@base_router.post(+ "/workflow/copilot/credential-generate",+ include_in_schema=False,+ response_model=WorkflowCopilotCredentialGenerateResult,+)+async def workflow_copilot_credential_generate(+ generate_request: WorkflowCopilotCredentialGenerateRequest,+ organization: Organization = Depends(org_auth_service.get_current_org_for_credential_routes),+ current_user_id: str | None = Depends(org_auth_service.get_current_user_id_or_none),+) -> WorkflowCopilotCredentialGenerateResult:+ """Answer a registration card by generating its password here and saving it as a credential, once per card.++ The password goes straight to the vault create: it is never returned, logged, or kept on the pause.+ """+ cache = getattr(app, "CACHE", None)+ if cache is None:+ raise HTTPException(+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,+ detail="Credential response not supported in this environment",+ )+ organization_id = organization.organization_id+ pause_ids = {+ "organization_id": organization_id,+ "workflow_copilot_chat_id": generate_request.workflow_copilot_chat_id,+ "turn_id": generate_request.turn_id,+ }+ try:+ claim = await claim_credential_generation(cache, resume_token=generate_request.resume_token, **pause_ids)+ except CredentialPauseRejection as rejection:+ raise HTTPException(status_code=rejection.status_code, detail=rejection.detail)+ spec = claim.registration+ claimed_at = datetime.now(UTC)+ credential_id: str | None = None+ outcome: Literal["rejected", "unknown"] = "unknown"+ try:+ password = generate_registration_password(spec.password_length, spec.charset)+ # Registered before anything can raise or log with it, so the log chain scrubs it from tracebacks too.+ skyvern_context.ensure_context().register_secret_value(password)+ data = CreateCredentialRequest(+ name=spec.credential_name,+ credential_type=CredentialType.PASSWORD,+ credential=NonEmptyPasswordCredential(username=spec.username, password=password),+ )+ credential_service = await asyncio.wait_for(+ prepare_credential_create(organization_id, data), _claim_seconds_left(claim.deadline)+ )+ except Exception as exc:+ outcome = "rejected"+ LOG.warning("copilot_credential_generation_rejected", error_type=type(exc).__name__)+ else:+ # A vault create cancelled midway can skip its own cleanup, so a timeout stops waiting but lets it finish.+ stored: asyncio.Future[Credential] = asyncio.get_running_loop().create_future()+ stored.add_done_callback(lambda future: future.cancelled() or future.exception())+ create = asyncio.create_task(+ _create_generated_credential(organization_id, data, current_user_id, credential_service, stored)+ )+ _DETACHED_CREDENTIAL_CREATES.add(create)+ create.add_done_callback(_DETACHED_CREDENTIAL_CREATES.discard)+ create_wait = max(0.0, _claim_seconds_left(claim.deadline) - _READBACK_RESERVE_SECONDS)+ try:+ credential_id = (await asyncio.wait_for(asyncio.shield(stored), create_wait)).credential_id+ except Exception as exc:+ vault_error = exc.__cause__ if isinstance(exc, HTTPException) else None+ if isinstance(vault_error, VaultHttpException) and 400 <= vault_error.status_code < 500:+ outcome = "rejected"+ LOG.warning("copilot_credential_generation_rejected", vault_status=vault_error.status_code)+ else:+ LOG.warning("copilot_credential_generation_unconfirmed", error_type=type(exc).__name__)+ if credential_id is None and outcome == "unknown":+ stored.add_done_callback(_log_late_create)+ try:+ credential_id = await asyncio.wait_for(+ _find_generated_credential(organization_id, data, current_user_id, claimed_at, credential_service),+ _claim_seconds_left(claim.deadline),+ )+ except Exception:+ LOG.warning("copilot_credential_generation_readback_failed", exc_info=True)+ LOG.info("copilot_credential_generation_readback", credential_id=credential_id)+ try:+ deadline = await finish_credential_generation(+ cache, claim=claim, credential_id=credential_id, outcome=outcome, **pause_ids+ )+ except Exception:+ if credential_id is None:+ raise+ LOG.warning("copilot_credential_generation_finish_failed", credential_id=credential_id, exc_info=True)+ deadline = None+ if credential_id is not None:+ LOG.info("copilot_credential_generation_created", credential_id=credential_id, connected=bool(deadline))+ return WorkflowCopilotCredentialGenerateResult(+ result="connected" if deadline else "created_not_connected",+ credential_id=credential_id,+ name=spec.credential_name,+ username=spec.username,+ )+ if deadline is None:+ raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Credential pause already resolved")+ return WorkflowCopilotCredentialGenerateResult(+ result=outcome, name=spec.credential_name, username=spec.username, expires_at=deadline+ )++@base_router.post("/workflow/copilot/clear-proposed-workflow", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT)diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 4e1dbc1a6..5154ee9c0 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -455,6 +455,9 @@ class WorkflowCopilotChatRequest(BaseModel):supports_credential_pause_recovery: bool = Field(False, description="The client restores pending credential cards from chat history.")+ supports_credential_generation: bool = Field(+ False, description="The client can render a registration card and post its Generate and save answer."+ )supports_credential_pause: bool = Field(False,description=(@@ -542,6 +545,28 @@ class WorkflowCopilotCredentialResponseRequest(BaseModel):credential_id: str | None = Field(None, description="Saved credential ID; required when action is 'connected'")+class WorkflowCopilotCredentialGenerateRequest(BaseModel):+ turn_id: str = Field(..., description="turn_id from the matching credential_required frame")+ workflow_copilot_chat_id: str = Field(..., description="chat ID from the matching credential_required frame")+ resume_token: str = Field(..., description="One-time resume token from the matching credential_required frame")+++class WorkflowCopilotCredentialGenerateResult(BaseModel):+ result: Literal["connected", "rejected", "unknown", "created_not_connected"] = Field(+ ...,+ description=(+ "'rejected' means nothing was saved; 'unknown' means the vault did not confirm the save, and the "+ "card stays open without Generate"+ ),+ )+ credential_id: str | None = Field(None, description="The saved credential, when one is known to exist")+ name: str | None = None+ username: str | None = None+ expires_at: datetime | None = Field(+ None, description="The reopened card's new deadline after 'rejected' or 'unknown'"+ )++class WorkflowCopilotCredentialResponseResult(BaseModel):result: Literal["accepted", "signed_in", "no_sign_in_found", "save_failed"]expires_at: datetime | None = Field(None, description="The pause's new deadline after 'signing_in'")@@ -961,6 +986,20 @@ class WorkflowCopilotTitleUpdate(BaseModel):timestamp: datetime = Field(..., description="Server timestamp")+class CredentialRegistration(BaseModel):+ username: str = Field(..., description="Username or email to register with, exactly as the user gave it")+ credential_name: str = Field(..., description="Name to save the new credential under")+ password_length: int = Field(24, description="Generated password length; 24 to 128")+ charset: Literal["alphanumeric", "alphanumeric_symbols"] = Field(+ "alphanumeric_symbols", description="Characters the site accepts in a password"+ )+++class WorkflowCopilotCredentialRegistration(CredentialRegistration):+ attempted: bool = Field(False, description="Generate and save was used; the card offers it once")+ outcome: Literal["rejected", "unknown"] | None = Field(None, description="Why an attempt did not connect")++class WorkflowCopilotCredentialRequiredUpdate(BaseModel):type: WorkflowCopilotStreamMessageType = Field(WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED, description="Message type"@@ -975,6 +1014,7 @@ class WorkflowCopilotCredentialRequiredUpdate(BaseModel):"login_credentials_unresolved","credential_missing_totp","credential_rejected_by_site",+ "credential_registration",] = Field(..., description="Typed signal that triggered the pause")message: str = Field(..., description="The agent's explanatory text at the moment of pausing")login_page_urls: list[str] = Field(default_factory=list, description="Candidate login page URLs, if known")@@ -988,6 +1028,9 @@ class WorkflowCopilotCredentialRequiredUpdate(BaseModel):None, description="The live browser the user may sign in to themselves; absent when the card does not offer it")signing_in: bool = Field(False, description="The user has started signing in themselves")+ registration: WorkflowCopilotCredentialRegistration | None = Field(+ None, description="Account details a Generate and save answer would store; never a password"+ )timestamp: datetime = Field(..., description="Server timestamp")diff --git a/skyvern/forge/sdk/services/route_authorization.py b/skyvern/forge/sdk/services/route_authorization.pyindex b10d47d65..44008a75d 100644--- a/skyvern/forge/sdk/services/route_authorization.py+++ b/skyvern/forge/sdk/services/route_authorization.py@@ -255,6 +255,7 @@ ROUTE_ACTIONS: Mapping[str, str] = {"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_chat_post": "workflow.update","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_clear_proposed_workflow": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_convert_yaml_to_blocks": "workflow_copilot.manage",+ "skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_credential_generate": "credential.create","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_credential_response": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_disable_auto_accept": "workflow_copilot.manage","skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_message_feedback": "workflow_copilot.manage",diff --git a/skyvern/forge/sdk/workflow/code_block_authorized_files.py b/skyvern/forge/sdk/workflow/code_block_authorized_files.pyindex c67dbd72f..7ce8a3f4b 100644--- a/skyvern/forge/sdk/workflow/code_block_authorized_files.py+++ b/skyvern/forge/sdk/workflow/code_block_authorized_files.py@@ -10,20 +10,22 @@ import reimport shutilimport statimport tempfile+import timefrom collections.abc import Awaitable, Callable, Mappingfrom contextlib import suppressfrom dataclasses import dataclassfrom functools import partialfrom pathlib import Path, PurePathfrom types import SimpleNamespace-from typing import Any, Protocol, cast+from typing import Any, Literal, Protocol, cast, get_argsimport structlog-from playwright.async_api import BrowserContext+from playwright.async_api import BrowserContext, Pagefrom skyvern.constants import SAVE_DOWNLOADED_FILES_TIMEOUTfrom skyvern.forge import appfrom skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.hashing import diagnostic_fingerprintfrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.webeye.browser_factory import resolve_artifact_path@@ -42,6 +44,20 @@ AUTHORIZED_FILE_CHANGED_ERROR = "The authorized file is unavailable or changed.AUTHORIZED_FILE_SHAPE_ERROR = ("attach_authorized_file accepts only a materialized file_url parameter or a download this run claimed.")+AUTHORIZED_FILE_MODE_ERROR = 'attach_authorized_file parameter \'mode\' must be "input" or "file_chooser".'+AUTHORIZED_FILE_CHOOSER_UNAVAILABLE_ERROR = (+ "File chooser mode is unavailable: this page or its browser worker does not support it, or the file is no "+ "longer authorized for this run (phase=chooser, trigger_state=not_attempted)."+)++AuthorizedFileAttachMode = Literal["input", "file_chooser"]+AuthorizedFileChooserPhase = Literal["chooser", "trigger", "attachment"]+AuthorizedFileTriggerState = Literal["not_attempted", "confirmed", "unknown"]+AUTHORIZED_FILE_ATTACH_MODES: tuple[AuthorizedFileAttachMode, ...] = get_args(AuthorizedFileAttachMode)+# One bound for arming, the single click, the chooser wait and set_files.+_CHOOSER_WAIT_SECONDS = 30+# Ends the bound this far before the block deadline, so the typed failure returns before the block is cut off.+_CHOOSER_DEADLINE_MARGIN_SECONDS = 1.0LOG = structlog.get_logger()@@ -58,14 +74,75 @@ class AuthorizedFileAccessError(Exception):pass-class FileInputTarget(Protocol):- def locator(self, selector: str) -> FileInputLocator: ...+class AuthorizedFileChooserError(AuthorizedFileAccessError):+ def __init__(+ self,+ phase: AuthorizedFileChooserPhase,+ trigger_state: AuthorizedFileTriggerState,+ message: str | None = None,+ ) -> None:+ self.phase = phase+ self.trigger_state = trigger_state+ suffix = "" if trigger_state == "not_attempted" else "; the trigger is not clicked again"+ super().__init__(+ message or f"The file chooser attach failed (phase={phase}, trigger_state={trigger_state}){suffix}."+ )class FileInputLocator(Protocol):async def set_input_files(self, files: dict[str, str | bytes]) -> None: ...+class ChooserTrigger(Protocol):+ async def click(self, *, timeout: float | None = None, trial: bool | None = None) -> None: ...+++class AttachLocator(FileInputLocator, ChooserTrigger, Protocol): ...+++class FileInputTarget(Protocol):+ def locator(self, selector: str) -> AttachLocator: ...+++class InlineAttach(Protocol):+ async def __call__(+ self, page: FileInputTarget, file: str | DownloadLike, selector: str, mode: str = "input"+ ) -> dict[str, str | int]: ...+++class FileChooserHandle(Protocol):+ async def set_files(self, files: dict[str, str | bytes], *, timeout: float | None = None) -> None: ...+++@dataclass(frozen=True, slots=True)+class FileChooserPins:+ """Unrecorded locator and listener methods taken from the raw page's class before authored code runs."""++ page: Page+ locate: Callable[[str], ChooserTrigger]+ listen: Callable[[str, Callable[[FileChooserHandle], None]], None]+ unlisten: Callable[[str, Callable[[FileChooserHandle], None]], None]+++def authorized_file_attach_modes(raw_page: Page) -> frozenset[AuthorizedFileAttachMode]:+ # The raw-CDP engine's FileChooser accepts host paths only, so it cannot take the verified in-memory bytes.+ if type(raw_page).__module__.startswith(("playwright.", "patchright.")):+ return frozenset(AUTHORIZED_FILE_ATTACH_MODES)+ return frozenset({"input"})+++def pin_file_chooser(raw_page: Page) -> FileChooserPins | None:+ if "file_chooser" not in authorized_file_attach_modes(raw_page):+ return None+ page_class = type(raw_page)+ return FileChooserPins(+ page=raw_page,+ locate=partial(page_class.locator, raw_page),+ listen=partial(page_class.on, raw_page),+ unlisten=partial(page_class.remove_listener, raw_page),+ )++@dataclass(frozen=True, slots=True)class MaterializedAuthorizedFile:workflow_run_id: str@@ -768,11 +845,48 @@ async def attach_materialized_authorized_file() -> dict[str, str | int]:# `locate` must be captured before authored code runs: inline code can reassign `page.locator`,# and a target resolved through the live page would hand the verified bytes to that object.+ _require_selector(selector)+ target = locate(selector)+ verified = await asyncio.to_thread(+ read_authorized_file,+ authorized_file,+ download_root=download_root,+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ max_bytes=max_bytes,+ )+ await target.set_input_files(_file_payload(verified))+ return {"filename": verified.filename, "size": verified.size}+++def _require_selector(selector: str) -> None:if not isinstance(selector, str) or not selector:raise AuthorizedFileAccessError("attach_authorized_file parameter 'selector' must be a non-empty selector string.")- target = locate(selector)+++def _file_payload(verified: AuthorizedFileBytes) -> dict[str, str | bytes]:+ mime_type = mimetypes.guess_type(verified.filename)[0] or "application/octet-stream"+ return {"name": verified.filename, "mimeType": mime_type, "buffer": verified.content}+++async def attach_authorized_file_via_chooser(+ locate: Callable[[str], ChooserTrigger],+ file_chooser: FileChooserPins | None,+ authorized_file: MaterializedAuthorizedFile,+ selector: str,+ *,+ download_root: str | Path,+ workflow_run_id: str,+ organization_id: str | None,+ max_bytes: int,+ deadline: float | None = None,+ passes_through: Callable[[Exception], bool] | None = None,+) -> dict[str, str | int]:+ """Verify the file, click ``selector`` once and fill the chooser it opens, before the monotonic ``deadline``.+ An error ``passes_through`` accepts is re-raised as is instead of becoming the typed chooser failure."""+ _require_selector(selector)verified = await asyncio.to_thread(read_authorized_file,authorized_file,@@ -781,8 +895,64 @@ async def attach_materialized_authorized_file(organization_id=organization_id,max_bytes=max_bytes,)- mime_type = mimetypes.guess_type(verified.filename)[0] or "application/octet-stream"- await target.set_input_files({"name": verified.filename, "mimeType": mime_type, "buffer": verified.content})+ if file_chooser is None:+ raise AuthorizedFileChooserError("chooser", "not_attempted", AUTHORIZED_FILE_CHOOSER_UNAVAILABLE_ERROR)+ bound: float = _CHOOSER_WAIT_SECONDS+ if deadline is not None:+ bound = min(bound, deadline - time.monotonic() - _CHOOSER_DEADLINE_MARGIN_SECONDS)+ if bound <= 0:+ raise AuthorizedFileChooserError("trigger", "not_attempted")+ # An agent UPLOAD_FILE that never saw its chooser leaves a listener on the page that would fill this one first.+ context = skyvern_context.current()+ if context is not None and context.pending_file_chooser is not None:+ if context.pending_file_chooser.page is file_chooser.page:+ context.cleanup_pending_file_chooser()+ loop = asyncio.get_running_loop()+ ends_at = loop.time() + bound+ opened: asyncio.Future[FileChooserHandle] = loop.create_future()++ def remaining_ms() -> float:+ # Playwright keeps a call running after its task is cancelled, so each call carries the bound itself.+ return max((ends_at - loop.time()) * 1000, 1)++ phase: AuthorizedFileChooserPhase = "chooser"+ trigger_state: AuthorizedFileTriggerState = "not_attempted"++ def on_file_chooser(chooser: FileChooserHandle) -> None:+ # A chooser the page opens before the real click is not the trigger's, so it never gets the bytes.+ if trigger_state != "not_attempted" and not opened.done():+ opened.set_result(chooser)++ try:+ async with asyncio.timeout_at(ends_at):+ # A listener rather than expect_file_chooser: a cancelled Playwright waiter keeps its listener,+ # and with it chooser interception, until its own timer fires.+ file_chooser.listen("filechooser", on_file_chooser)+ try:+ phase = "trigger"+ # Trial runs the actionability and strictness checks without dispatching, so a trigger+ # that cannot be clicked fails before anything reaches the page.+ await file_chooser.locate(selector).click(trial=True, timeout=remaining_ms())+ trigger_state = "unknown"+ await locate(selector).click(timeout=remaining_ms())+ trigger_state = "confirmed"+ phase = "chooser"+ chooser = await opened+ phase = "attachment"+ await chooser.set_files(_file_payload(verified), timeout=remaining_ms())+ finally:+ with suppress(Exception):+ file_chooser.unlisten("filechooser", on_file_chooser)+ except Exception as exc:+ if passes_through is not None and passes_through(exc):+ raise+ LOG.warning(+ "codeblock.authorized_file_chooser_failed",+ phase=phase,+ trigger_state=trigger_state,+ error_type=type(exc).__name__,+ )+ raise AuthorizedFileChooserError(phase, trigger_state) from Nonereturn {"filename": verified.filename, "size": verified.size}@@ -797,9 +967,11 @@ def bind_inline_attach_authorized_file(max_bytes: int,download_run_id: str | None = None,download_log: BlockDownloadLog | None = None,- locate: Callable[[str], FileInputLocator] | None = None,+ locate: Callable[[str], AttachLocator] | None = None,registered_downloads: RegisteredDownloadSource | None = None,-) -> Callable[[FileInputTarget, str | DownloadLike, str], Awaitable[dict[str, str | int]]]:+ file_chooser: FileChooserPins | None = None,+ deadline: float | None = None,+) -> InlineAttach:authorized_by_path = {inline_authorized_file_path(item, download_root=download_root): itemfor item in materializations.values()@@ -826,9 +998,12 @@ def bind_inline_attach_authorized_file(page: FileInputTarget,file: str | DownloadLike,selector: str,+ mode: str = "input",) -> dict[str, str | int]:if page is not expected_page:raise AuthorizedFileAccessError("attach_authorized_file requires the current CodeBlock page.")+ if type(mode) is not str or mode not in AUTHORIZED_FILE_ATTACH_MODES:+ raise AuthorizedFileAccessError(AUTHORIZED_FILE_MODE_ERROR)# Capability, not class identity: a raw-CDP download is not a Playwright ``Download``, and the# bytes come from the run's own download directory either way, never from this object.if is_download_like(file):@@ -867,6 +1042,18 @@ def bind_inline_attach_authorized_file(authorized_file = resolvedif locate is None:raise AuthorizedFileAccessError("attach_authorized_file requires the current CodeBlock page.")+ if mode == "file_chooser":+ return await attach_authorized_file_via_chooser(+ locate,+ file_chooser,+ authorized_file,+ selector,+ download_root=download_root,+ workflow_run_id=run_key,+ organization_id=organization_id,+ max_bytes=max_bytes,+ deadline=deadline,+ )return await attach_materialized_authorized_file(locate,authorized_file,@@ -884,6 +1071,7 @@ async def unbound_attach_authorized_file(page: FileInputTarget,file: str,selector: str,+ mode: str = "input",) -> dict[str, str | int]:raise AuthorizedFileAccessError("attach_authorized_file is only available for a materialized file_url parameter during a workflow run."diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 343246dca..f58dddf79 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -208,6 +208,7 @@ from skyvern.forge.sdk.workflow.code_block_authorized_files import (bind_inline_attach_authorized_file,capture_authorized_file,inline_authorized_file_path,+ pin_file_chooser,unbound_attach_authorized_file,)from skyvern.forge.sdk.workflow.code_block_safety import BLOCKED_ATTRS as CODE_BLOCK_BLOCKED_ATTRS@@ -6787,6 +6788,13 @@ class CodeBlock(Block):download_log=download_log,locate=page._pinned_locator() if isinstance(page, RecordingPage) else None,registered_downloads=registered_downloads,+ file_chooser=pin_file_chooser(_raw_code_block_page(page)),+ # Taken before execute_user_function_with_timeout starts its clock, so it errs early.+ deadline=(+ monotonic() + settings.CODE_BLOCK_EXECUTION_TIMEOUT_SECONDS+ if settings.CODE_BLOCK_EXECUTION_TIMEOUT_SECONDS > 0+ else None+ ),)safe_vars["open_page"] = _bind_code_block_open_page(page, opened_pages if opened_pages is not None else [], browser_state@@ -9996,7 +10004,7 @@ async def wrapper({default_args}):failure_page=failed_page,workflow_run_context=workflow_run_context,redaction_parameters=serialized_parameter_values,- opened_pages=opened_pages,+ opened_pages=[*opened_pages, *recording_page._claimed_popups()],)if inline_failure_page_state:from skyvern.forge.sdk.workflow.models.code_block_recorder import append_failure_page_statediff --git a/skyvern/forge/sdk/workflow/models/code_block_recorder.py b/skyvern/forge/sdk/workflow/models/code_block_recorder.pyindex dcca11172..ebf09125b 100644--- a/skyvern/forge/sdk/workflow/models/code_block_recorder.py+++ b/skyvern/forge/sdk/workflow/models/code_block_recorder.py@@ -462,6 +462,7 @@ class _Recorder:# has to be the same object, or identity comparisons fail and the worker hands out a# second handle for the page it already registered.self.page_proxies: dict[int, tuple[Any, Any]] = {}+ self.claimed_popups: dict[int, Page] = {}self.failure_operation_generation = 0self._next_action_order = 0self._on_action = on_action@@ -1244,6 +1245,13 @@ class RecordingPage:def _wrap_page(self, page: Any) -> Any:return self.__recording_page(page)+ def __claim_popup(self, page: Page) -> RecordingPage:+ # Playwright still yields a popup that closed before the block read it; it is wrapped but not+ # claimed, so failure evidence never names a tab that is already gone.+ if not page.is_closed():+ self.__recorder.claimed_popups[id(page)] = page+ return self.__recording_page(page)+def __recording_page(self, page: Any) -> Any:if page is self.__page:return self@@ -1317,6 +1325,9 @@ class RecordingPage:def failure_locator(self, exception: BaseException) -> Locator | None:return self.__recorder.failed_locator if self.__recorder.failed_locator_exception is exception else None+ def _claimed_popups(self) -> list[Page]:+ return list(self.__recorder.claimed_popups.values())+def failure_page(self, exception: BaseException) -> Page | None:"""The raw page the call that raised ``exception`` ran on, whether through the page, one ofits frames or a locator; None when the exception did not come from a recorded call."""@@ -1499,9 +1510,7 @@ class RecordingPage:generation,self.__page,event_value=(- self.__recording_page- if name == "expect_popup"- else partial(_wrap_if_page, self.__recording_page)+ self.__claim_popup if name == "expect_popup" else partial(_wrap_if_page, self.__recording_page)),)diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.pyindex 6c0a3d8c1..ee549d24a 100644--- a/skyvern/forge/sdk/workflow/models/workflow.py+++ b/skyvern/forge/sdk/workflow/models/workflow.py@@ -1,3 +1,5 @@+import hashlib+import jsonfrom datetime import datetimefrom enum import StrEnumfrom typing import Any, List@@ -365,6 +367,8 @@ class WorkflowRun(BaseModel):reuse_bound_key: str | None = Field(default=None, exclude=True)# Internal routing: the worker queue the run was dispatched to. Never an API field.task_queue: str | None = Field(default=None, exclude=True)+ # Digest of the definition the run was created against; the saved row can be overwritten in place later.+ workflow_definition_sha256: str | None = Field(default=None, exclude=True)debug_session_id: str | None = Nonestatus: WorkflowRunStatusattempt: int = Field(default=1, description="One-based number of the current workflow run attempt")@@ -436,6 +440,14 @@ class WorkflowRun(BaseModel):return self.debug_session_id is not None+# Stored when attempts of one run executed different definitions; it never equals a real digest.+MIXED_RUN_DEFINITION_DIGEST = "mixed"+++def workflow_definition_sha256(definition: WorkflowDefinition) -> str:+ return hashlib.sha256(json.dumps(definition.model_dump(mode="json"), sort_keys=True).encode("utf-8")).hexdigest()++def start_hold_reason(*, sequential_key: str | None, depends_on_workflow_run_id: str | None) -> str:"""Why a run may wait by design before it starts: a sequential lane, a dependency, or neither."""if sequential_key:diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex d7021ea72..74d35ece2 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -226,6 +226,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (resolve_reuse_browser_session,should_acquire_reused_session,start_hold_reason,+ workflow_definition_sha256,)from skyvern.forge.sdk.workflow.private_settings import (resolve_cdp_connect_headers,@@ -8127,7 +8128,12 @@ class WorkflowService:# The run_with field records what the user requested (e.g. "code"),# not whether a script was actually found. Execution mode is determined# separately by is_script_run and script_mode below.- await self.mark_workflow_run_as_running(workflow_run_id=workflow_run_id, run_with=workflow_run.run_with)+ # The row can be rewritten while the run is queued, so re-stamp the digest from what executes.+ await self.mark_workflow_run_as_running(+ workflow_run_id=workflow_run_id,+ run_with=workflow_run.run_with,+ workflow_definition_sha256=workflow_definition_sha256(workflow.workflow_definition),+ )# Set script_mode on context so downstream code can skip expensive LLM calls# Only enable when we actually have a script to run@@ -11562,6 +11568,10 @@ class WorkflowService:organization_id=organization_id,)+ if workflow is None:+ workflow = await self.get_workflow(workflow_id=workflow_id)+ definition_sha256 = workflow_definition_sha256(workflow.workflow_definition)+# Sample the kill switch exactly once, before request-time precedence. This persisted# admission decision is authoritative: later flag changes stop new runs but never revoke an# in-flight run or strip the forced-session fallback that was selected at admission.@@ -11570,8 +11580,6 @@ class WorkflowService:persisted_reuse_bound_key: str | None = Nonepersisted_reuse_browser_session = workflow_request.reuse_browser_sessionif not browser_session_id:- if workflow is None:- workflow = await self.get_workflow(workflow_id=workflow_id)configured_reuse = should_acquire_reused_session(browser_session_id=None,start_fresh_browser=workflow_request.start_fresh_browser,@@ -11643,6 +11651,7 @@ class WorkflowService:ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,copilot_session_id=copilot_session_id,created_by=created_by,+ workflow_definition_sha256=definition_sha256,)# A block run creates its block-run rows only after setup, so the caller's intent# is the only block-scoped signal enrolment can see here.@@ -11717,6 +11726,7 @@ class WorkflowService:ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,copilot_session_id=copilot_session_id,created_by=created_by,+ workflow_definition_sha256=definition_sha256,)if not block_scoped:await ensure_attempt_row(@@ -13540,7 +13550,12 @@ class WorkflowService:LOG.exception("Failed to cascade child entity status", workflow_run_id=workflow_run_id)return workflow_run- async def mark_workflow_run_as_running(self, workflow_run_id: str, run_with: str | None = None) -> WorkflowRun:+ async def mark_workflow_run_as_running(+ self,+ workflow_run_id: str,+ run_with: str | None = None,+ workflow_definition_sha256: str | None = None,+ ) -> WorkflowRun:# Conditional UPDATE refuses to resurrect a finalized wr — prevents the# cleanup cron from racing with re-entry paths and stomping timed_out# back to running.@@ -13548,6 +13563,7 @@ class WorkflowService:workflow_run_id=workflow_run_id,status=WorkflowRunStatus.running,run_with=run_with,+ workflow_definition_sha256=workflow_definition_sha256,)if workflow_run is None:existing = await app.DATABASE.workflow_runs.get_workflow_run(diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.pyindex d36f2f4ba..8fd3d7270 100644--- a/skyvern/services/task_v2_service.py+++ b/skyvern/services/task_v2_service.py@@ -61,7 +61,13 @@ from skyvern.forge.sdk.workflow.models.block import (UrlBlock,)from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE, ContextParameter-from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRequestBody, WorkflowRun, WorkflowRunStatus+from skyvern.forge.sdk.workflow.models.workflow import (+ Workflow,+ WorkflowRequestBody,+ WorkflowRun,+ WorkflowRunStatus,+ workflow_definition_sha256,+)from skyvern.schemas.proxy_location import runtime_proxy_locationfrom skyvern.schemas.runs import (ProxyLocationInput,@@ -809,9 +815,11 @@ async def run_task_v2_helper(task_v2 = await app.DATABASE.observer.update_task_v2(task_v2_id=task_v2_id, organization_id=organization_id, status=TaskV2Status.running)- await app.WORKFLOW_SERVICE.mark_workflow_run_as_running(workflow_run_id=workflow_run.workflow_run_id)-workflow = await app.WORKFLOW_SERVICE.get_workflow(workflow_id=workflow_run.workflow_id)+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_running(+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_definition_sha256=workflow_definition_sha256(workflow.workflow_definition),+ )await _set_up_workflow_context(workflow, workflow_run_id, organization)user_prompt = task_v2.promptdiff --git a/skyvern/webeye/skycdp/facade/artifacts.py b/skyvern/webeye/skycdp/facade/artifacts.pyindex ef32000e6..751fbfdee 100644--- a/skyvern/webeye/skycdp/facade/artifacts.py+++ b/skyvern/webeye/skycdp/facade/artifacts.py@@ -114,7 +114,8 @@ class FileChooser:"""return self._multiple- async def set_files(self, files: str | list[str]) -> None:+ async def set_files(self, files: str | list[str], *, timeout: float | None = None) -> None:+ # Playwright's signature has `timeout` and production passes it; one CDP call has nothing to time out.paths = [files] if isinstance(files, str) else list(files)missing = [path for path in paths if not Path(path).is_file()]if missing:diff --git a/tests/conftest.py b/tests/conftest.pyindex 151b28fac..ba2f7ecd1 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -1,17 +1,21 @@"""Fixtures shared by every suite."""import inspect+import io+import loggingfrom collections.abc import Callable, Iteratorfrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMockimport pytest+import structlog-from skyvern.forge import app+from skyvern.forge import app, forge_app_initializer, request_loggingfrom skyvern.forge.sdk.core import organization_age_cachefrom skyvern.forge.sdk.experimentation.code_block_ai_fallback import CODE_BLOCK_AI_FALLBACK_FLAGfrom skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProvider+from skyvern.forge.sdk.forge_log import setup_loggerfrom skyvern.forge.sdk.workflow.models.block import CodeBlockfrom skyvern.services import organization_log_scope@@ -26,6 +30,28 @@ def _isolate_organization_age_cache() -> Iterator[None]:organization_log_scope._warmups_in_flight.clear()+def rendered_log_stream(monkeypatch: pytest.MonkeyPatch) -> Iterator[io.StringIO]:+ """Every log line as production renders it (JSON, raw request logging on), written to a stream."""+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS", True)+ monkeypatch.setattr(request_logging.settings, "JSON_LOGGING", True)+ root_logger = logging.getLogger()+ saved_handlers = root_logger.handlers[:]+ saved_structlog_config = structlog.get_config()+ setup_logger()+ # create_api_app configures logging once per process; keep it from replacing this handler.+ monkeypatch.setattr(forge_app_initializer, "_SERVER_LOGGING_CONFIGURED", True)+ stream = io.StringIO()+ handler = root_logger.handlers[0]+ assert isinstance(handler, logging.StreamHandler)+ handler.setStream(stream)+ try:+ yield stream+ finally:+ root_logger.handlers[:] = saved_handlers+ structlog.configure(**saved_structlog_config)++class ForcedSinkFailure(RuntimeError):"""Stands in for a real sink error when a test forces a side effect to fail."""diff --git a/tests/unit/_file_chooser_fakes.py b/tests/unit/_file_chooser_fakes.pynew file mode 100644index 000000000..049371e96--- /dev/null+++ b/tests/unit/_file_chooser_fakes.py@@ -0,0 +1,126 @@+"""A page whose upload button creates its file input only when clicked, so only a chooser listener sees it."""++from __future__ import annotations++import asyncio+from collections.abc import Callable++from skyvern.forge.sdk.workflow.code_block_authorized_files import FileChooserHandle++ChooserListener = Callable[[FileChooserHandle], None]+++async def _drive(timeout: float | None) -> None:+ if timeout is None:+ await asyncio.Event().wait()+ else:+ await asyncio.sleep(timeout / 1000)+++async def _outlive_cancel(page: FakeChooserPage, timeout: float | None) -> None:+ # Like Playwright's driver: cancelling the caller leaves the call running until its own timeout.+ driver_call = asyncio.ensure_future(_drive(timeout))+ page.driver_calls.append(driver_call)+ await asyncio.shield(driver_call)+++class FakeChooserHandle:+ def __init__(self, page: FakeChooserPage, into: list[dict[str, str | bytes]] | None = None) -> None:+ self._page = page+ self._into = page.chosen if into is None else into++ async def set_files(self, files: dict[str, str | bytes], *, timeout: float | None = None) -> None:+ self._page.set_files_timeouts.append(timeout)+ if self._page.set_files_outlives_cancel:+ await _outlive_cancel(self._page, timeout)+ if self._page.set_files_error is not None:+ raise self._page.set_files_error+ self._into.append(files)+++class FakeChooserTrigger:+ def __init__(self, page: FakeChooserPage, selector: str) -> None:+ self.page, self.selector = page, selector++ async def click(self, *, timeout: float | None = None, trial: bool | None = None) -> None:+ page = self.page+ page.clicks.append((self.selector, bool(trial)))+ page.click_timeouts.append(timeout)+ if trial:+ if page.trial_error is not None:+ raise page.trial_error+ if page.trial_opens_chooser:+ for handler in list(page.listeners.get("filechooser", [])):+ handler(FakeChooserHandle(page, into=page.stray_chosen))+ return+ if page.click_blocks:+ await asyncio.Event().wait()+ if page.click_outlives_cancel:+ await _outlive_cancel(page, timeout)+ if page.click_error is not None:+ raise page.click_error+ if page.opens_chooser:+ for handler in list(page.listeners.get("filechooser", [])):+ handler(FakeChooserHandle(page))++ async def set_input_files(self, files: dict[str, str | bytes]) -> None:+ self.page.input_files.append(files)+++class FakeChooserPage:+ def __init__(+ self,+ *,+ opens_chooser: bool = True,+ trial_error: Exception | None = None,+ click_error: Exception | None = None,+ set_files_error: Exception | None = None,+ click_blocks: bool = False,+ click_outlives_cancel: bool = False,+ set_files_outlives_cancel: bool = False,+ trial_opens_chooser: bool = False,+ ) -> None:+ self.opens_chooser = opens_chooser+ self.trial_error = trial_error+ self.click_error = click_error+ self.set_files_error = set_files_error+ self.click_blocks = click_blocks+ self.click_outlives_cancel = click_outlives_cancel+ self.set_files_outlives_cancel = set_files_outlives_cancel+ self.trial_opens_chooser = trial_opens_chooser+ self.clicks: list[tuple[str, bool]] = []+ self.click_timeouts: list[float | None] = []+ self.set_files_timeouts: list[float | None] = []+ self.driver_calls: list[asyncio.Future[None]] = []+ self.chosen: list[dict[str, str | bytes]] = []+ self.stray_chosen: list[dict[str, str | bytes]] = []+ self.input_files: list[dict[str, str | bytes]] = []+ self.listeners: dict[str, list[ChooserListener]] = {}++ def locator(self, selector: str) -> FakeChooserTrigger:+ return FakeChooserTrigger(self, selector)++ def on(self, event: str, handler: ChooserListener) -> None:+ self.listeners.setdefault(event, []).append(handler)++ def remove_listener(self, event: str, handler: ChooserListener) -> None:+ self.listeners[event].remove(handler)++ @property+ def dispatched_clicks(self) -> int:+ return sum(1 for _, trial in self.clicks if not trial)++ @property+ def live_listeners(self) -> int:+ return len(self.listeners.get("filechooser", []))+++# Chooser mode is offered by page class family, so the fakes claim the module of the page they stand in for.+FakeChooserPage.__module__ = "playwright.async_api._generated"+++class FakeRawCDPChooserPage(FakeChooserPage):+ pass+++FakeRawCDPChooserPage.__module__ = "skyvern.webeye.skycdp.facade.page"diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.pyindex 1135be0ac..279a3c4ed 100644--- a/tests/unit/copilot_test_helpers.py+++ b/tests/unit/copilot_test_helpers.py@@ -57,9 +57,11 @@ from skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSessionfrom skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest, WorkflowCopilotTitleUpdatefrom skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock+from skyvern.forge.sdk.workflow.models.block import CodeBlockfrom skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameter, WorkflowParameterTypefrom skyvern.forge.sdk.workflow.models.workflow import (Workflow,+ WorkflowDefinition,WorkflowRun,WorkflowRunOutputParameter,WorkflowRunParameter,@@ -236,6 +238,7 @@ def harness_run(copilot_session_id: str | None = None,workflow_permanent_id: str = "wpid-1",organization_id: str = "org-1",+ workflow_definition_sha256: str | None = None,) -> SimpleNamespace:"""Naive-UTC created_at, as the database returns it."""return SimpleNamespace(@@ -249,6 +252,7 @@ def harness_run(copilot_session_id=copilot_session_id,failure_reason=None,browser_session_id="pbs-1",+ workflow_definition_sha256=workflow_definition_sha256,)@@ -266,14 +270,23 @@ def install_get_run_results_harness(carried_successful_run_id: str | None = None,carried_run_id: str | None = None,heal_episodes: list[HealEpisode] | None = None,+ run_definition_sha256: str | None = None,) -> SimpleNamespace:"""Stub the collaborators ``_get_run_results`` reaches and return the ctx to call it with; the run pool is``wr-1`` plus ``other_runs``, and run lookup and history listing honor their arguments."""- pool = [harness_run("wr-1", status=run_status), *(other_runs or [])]+ pool = [+ harness_run("wr-1", status=run_status, workflow_definition_sha256=run_definition_sha256),+ *(other_runs or []),+ ]workflow = SimpleNamespace(+ workflow_id="wf-1",+ workflow_permanent_id="wpid-1",+ version=1,created_by=None,modified_at=HARNESS_RUN_CREATED_AT,- workflow_definition=SimpleNamespace(parameters=workflow_parameters or [], blocks=[]),+ workflow_definition=SimpleNamespace(+ parameters=[SimpleNamespace(**parameter) for parameter in workflow_parameters or []], blocks=[]+ ),)async def get_workflow_run(workflow_run_id: str, organization_id: str | None = None) -> SimpleNamespace | None:@@ -349,7 +362,86 @@ def install_get_run_results_harness(proposal_workflow_run_id=None,dispatched_run_ids_this_turn=set(),seeded_only_labels_by_run_id={},+ workflow_yaml=None,+ staged_workflow_yaml=None,+ )+++def historical_code_version(+ blocks: Mapping[str, str],+ *,+ parameter_keys: Sequence[str] = ("applicant_name",),+ default_value: str = "Fixture default",+ workflow_id: str = "wf-1",+ modified_at: datetime = HARNESS_RUN_CREATED_AT,+) -> SimpleNamespace:+ """A saved version of ``wpid-1`` holding one CodeBlock per ``blocks`` label, as a finished harness run read it."""+ return SimpleNamespace(+ workflow_id=workflow_id,+ workflow_permanent_id="wpid-1",+ version=3,+ created_by=None,+ modified_at=modified_at,+ workflow_definition=WorkflowDefinition(+ parameters=[+ WorkflowParameter(+ workflow_parameter_id=f"wp_{key}",+ workflow_parameter_type=WorkflowParameterType.STRING,+ key=key,+ default_value=default_value,+ workflow_id=workflow_id,+ created_at=HARNESS_RUN_CREATED_AT,+ modified_at=HARNESS_RUN_CREATED_AT,+ )+ for key in parameter_keys+ ],+ blocks=[+ CodeBlock(+ label=label,+ code=code,+ output_parameter=OutputParameter(+ output_parameter_id=f"op_{label}",+ key=f"{label}_output",+ workflow_id=workflow_id,+ created_at=HARNESS_RUN_CREATED_AT,+ modified_at=HARNESS_RUN_CREATED_AT,+ ),+ )+ for label, code in blocks.items()+ ],+ ),+ )+++def install_historical_run(+ monkeypatch: pytest.MonkeyPatch,+ version: SimpleNamespace | None,+ *,+ failed_label: str,+ failing_line: int = 1,+ run_definition_sha256: str | None = None,+) -> None:+ """Harness run ``wr-1`` failed at ``failing_line`` of ``failed_label`` while executing ``version``."""++ async def stamp_failing_line(+ _rows: object, results: list[dict[str, Any]], _org: str, include_completed: bool = False+ ) -> None:+ results[0]["action_trace"] = [{"action": "NULL_ACTION", "status": "failed", "code_line": failing_line}]++ block = run_result_block_row(+ failed_label, "failed", failure_reason="NameError: a name is not defined", error_codes=["user_code_error"]+ )+ block.created_at = HARNESS_RUN_CREATED_AT+ block.parent_workflow_run_block_id = None+ block.current_index = None+ block.current_value = None+ install_get_run_results_harness(+ monkeypatch,+ blocks=[block],+ attach_action_traces=stamp_failing_line,+ run_definition_sha256=run_definition_sha256,)+ run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run.return_value = versiondef run_result_action_row(@@ -490,8 +582,8 @@ async def install_run_blocks_harness(polled_run = _fake_workflow_run(status=polled_status)- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, Any, Any]:- return polled_run, now, now+ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, Any]:+ return polled_run, nowmonkeypatch.setattr(run_execution_module, "_read_progress_sources", _read_progress)monkeypatch.setattr(run_execution_module, "RUN_BLOCKS_POLL_INTERVAL_SECONDS", 0)diff --git a/tests/unit/forge/sdk/db/test_workflow_attribution_columns.py b/tests/unit/forge/sdk/db/test_workflow_attribution_columns.pyindex c9b164a7b..a1702102a 100644--- a/tests/unit/forge/sdk/db/test_workflow_attribution_columns.py+++ b/tests/unit/forge/sdk/db/test_workflow_attribution_columns.py@@ -11,6 +11,11 @@ from sqlalchemy.ext.asyncio import create_async_enginefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.db.agent_db import AgentDBfrom skyvern.forge.sdk.db.models import Base+from skyvern.forge.sdk.workflow.models.workflow import (+ MIXED_RUN_DEFINITION_DIGEST,+ WorkflowRunStatus,+ workflow_definition_sha256,+)@pytest_asyncio.fixture@@ -218,3 +223,104 @@ async def test_workflow_never_touched_by_copilot_is_not_copilot_authored(agent_dworkflow_permanent_id=workflow.workflow_permanent_id,organization_id=org_id,)+++async def test_run_definition_digest_survives_reload_and_metadata_edits_but_not_a_rewrite(+ agent_db: AgentDB, org_id: str+) -> None:+ workflow = await agent_db.workflows.create_workflow(+ title="wf",+ workflow_definition={"parameters": [], "blocks": []},+ organization_id=org_id,+ )+ run = await agent_db.workflow_runs.create_workflow_run(+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_id=workflow.workflow_id,+ organization_id=org_id,+ workflow_definition_sha256=workflow_definition_sha256(workflow.workflow_definition),+ )+ stored = await agent_db.workflow_runs.get_workflow_run(run.workflow_run_id, organization_id=org_id)+ assert stored is not None and stored.workflow_definition_sha256 is not None++ async def digest_of_row_the_run_reads() -> str:+ row = await agent_db.workflows.get_workflow_for_workflow_run(run.workflow_run_id, organization_id=org_id)+ assert row is not None+ return workflow_definition_sha256(row.workflow_definition)++ assert await digest_of_row_the_run_reads() == stored.workflow_definition_sha256+ await agent_db.workflows.update_workflow(workflow_id=workflow.workflow_id, organization_id=org_id, title="renamed")+ assert await digest_of_row_the_run_reads() == stored.workflow_definition_sha256+ await agent_db.workflows.update_workflow(+ workflow_id=workflow.workflow_id,+ organization_id=org_id,+ workflow_definition={"parameters": [], "blocks": [], "workflow_system_prompt": "rewritten in place"},+ )+ assert await digest_of_row_the_run_reads() != stored.workflow_definition_sha256+++ "first_attempt_runs_original", [True, False], ids=["rewritten-between-retries", "rewritten-while-queued"]+)+async def test_a_run_whose_attempts_saw_different_definitions_never_matches_its_row(+ agent_db: AgentDB, org_id: str, first_attempt_runs_original: bool+) -> None:+ original = {"parameters": [], "blocks": []}+ workflow = await agent_db.workflows.create_workflow(+ title="wf", workflow_definition=original, organization_id=org_id+ )+ original_digest = workflow_definition_sha256(workflow.workflow_definition)+ run = await agent_db.workflow_runs.create_workflow_run(+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_id=workflow.workflow_id,+ organization_id=org_id,+ workflow_definition_sha256=original_digest,+ )++ async def start_attempt(digest: str) -> str | None:+ started = await agent_db.workflow_runs.update_workflow_run_if_not_final(+ workflow_run_id=run.workflow_run_id, status=WorkflowRunStatus.running, workflow_definition_sha256=digest+ )+ assert started is not None+ return started.workflow_definition_sha256++ if first_attempt_runs_original:+ assert await start_attempt(original_digest) == original_digest+ rewritten = await agent_db.workflows.update_workflow(+ workflow_id=workflow.workflow_id,+ organization_id=org_id,+ workflow_definition={**original, "workflow_system_prompt": "rewritten in place"},+ )+ stored = await start_attempt(workflow_definition_sha256(rewritten.workflow_definition))++ row = await agent_db.workflows.get_workflow_for_workflow_run(run.workflow_run_id, organization_id=org_id)+ assert row is not None+ assert stored == MIXED_RUN_DEFINITION_DIGEST+ assert workflow_definition_sha256(row.workflow_definition) != stored+++async def test_a_run_created_before_digests_is_never_stamped_by_a_later_attempt(agent_db: AgentDB, org_id: str) -> None:+ # Its earlier attempts' block rows may describe a definition nobody recorded, so it stays on the modified_at rule.+ workflow = await agent_db.workflows.create_workflow(+ title="wf", workflow_definition={"parameters": [], "blocks": []}, organization_id=org_id+ )+ run = await agent_db.workflow_runs.create_workflow_run(+ workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_id=workflow.workflow_id,+ organization_id=org_id,+ )+ rewritten = await agent_db.workflows.update_workflow(+ workflow_id=workflow.workflow_id,+ organization_id=org_id,+ workflow_definition={"parameters": [], "blocks": [], "workflow_system_prompt": "rewritten before the retry"},+ )++ retried = await agent_db.workflow_runs.update_workflow_run_if_not_final(+ workflow_run_id=run.workflow_run_id,+ status=WorkflowRunStatus.running,+ workflow_definition_sha256=workflow_definition_sha256(rewritten.workflow_definition),+ )++ assert retried is not None and retried.workflow_definition_sha256 is Nonediff --git a/tests/unit/test_code_block_authorized_files.py b/tests/unit/test_code_block_authorized_files.pyindex 2e7eab1fd..a2281838e 100644--- a/tests/unit/test_code_block_authorized_files.py+++ b/tests/unit/test_code_block_authorized_files.py@@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMockimport pytestfrom playwright.async_api import Download+from playwright.async_api import Error as PlaywrightErrorfrom structlog.testing import capture_logsfrom skyvern.config import settings@@ -23,17 +24,26 @@ from skyvern.forge.sdk.artifact.models import ArtifactTypefrom skyvern.forge.sdk.artifact.storage import base as storage_base_modulefrom skyvern.forge.sdk.artifact.storage import local as local_storage_modulefrom skyvern.forge.sdk.artifact.storage.local import LocalStorage+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import PendingFileChooserListener, SkyvernContextfrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.workflow import code_block_authorized_filesfrom skyvern.forge.sdk.workflow.code_block_authorized_files import (+ AUTHORIZED_FILE_CHOOSER_UNAVAILABLE_ERROR,AuthorizedFileAccessError,+ AuthorizedFileChooserError,AuthorizedFileMaterializationFailure,BlockDownloadLog,+ FileChooserHandle,+ InlineAttach,+ MaterializedAuthorizedFile,RegisteredDownloadIdentity,RegisteredDownloadSource,+ authorized_file_attach_modes,bind_inline_attach_authorized_file,capture_authorized_file,capture_claimed_download,+ pin_file_chooser,read_authorized_file,)from skyvern.forge.sdk.workflow.models.block import CodeBlock, _registered_download_source@@ -42,6 +52,7 @@ from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, Parametfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.browser_artifacts import DownloadBindingfrom skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptor+from tests.unit._file_chooser_fakes import FakeChooserPage, FakeRawCDPChooserPagefrom tests.unit.conftest import SESSION_DOWNLOAD_BYTES as _SESSION_BYTESfrom tests.unit.conftest import make_claimed_download_mockfrom tests.unit.conftest import registered_download_row as _registered_row@@ -1606,3 +1617,377 @@ async def test_only_a_signed_remote_binding_snapshots_the_registered_downloads(m("certificate.pdf", hashlib.sha256(_SESSION_BYTES).hexdigest()): "certificate.pdf"}assert probe.await_count == 1+++def _bind_chooser_attach(+ raw_page: FakeChooserPage,+ path: Path,+ materialized: MaterializedAuthorizedFile,+ *,+ workflow_run_id: str = "wr_authorized",+ organization_id: str = "org_authorized",+ max_bytes: int = 1024,+ deadline: float | None = None,+) -> tuple[RecordingPage, InlineAttach]:+ recording_page = RecordingPage(raw_page) # type: ignore[arg-type]+ attach = bind_inline_attach_authorized_file(+ recording_page, # type: ignore[arg-type]+ {"resume": materialized, "cover_letter": AuthorizedFileMaterializationFailure()},+ {"resume": str(path), "cover_letter": "https://invalid.example/cover.pdf"},+ download_root=path.parents[1],+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ max_bytes=max_bytes,+ locate=recording_page._pinned_locator(),+ file_chooser=pin_file_chooser(raw_page),+ deadline=deadline,+ )+ return recording_page, attach+++def _recorded_types(recording_page: RecordingPage) -> list[ActionType]:+ return [action.action_type for action in recording_page.recorded_actions()]+++async def test_chooser_mode_clicks_the_trigger_once_and_hands_the_verified_bytes_to_the_chooser(+ authorized_file,+) -> None:+ path, materialized = authorized_file+ raw_page = FakeChooserPage()+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)++ receipt = await attach(recording_page, str(path), "#choose-file", mode="file_chooser")++ assert receipt == {"filename": "resume.pdf", "size": len(b"retained authorized bytes")}+ assert raw_page.clicks == [("#choose-file", True), ("#choose-file", False)]+ assert [files["buffer"] for files in raw_page.chosen] == [b"retained authorized bytes"]+ assert raw_page.input_files == []+ # The trial click goes through the unrecorded raw locator, so the timeline shows the one real click.+ assert _recorded_types(recording_page) == [ActionType.CLICK]+ assert raw_page.live_listeners == 0+++async def test_a_chooser_the_page_opens_before_the_real_click_never_receives_the_bytes(authorized_file) -> None:+ path, materialized = authorized_file+ raw_page = FakeChooserPage(trial_opens_chooser=True)+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)++ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")++ assert raw_page.stray_chosen == []+ assert [files["buffer"] for files in raw_page.chosen] == [b"retained authorized bytes"]+ assert raw_page.dispatched_clicks == 1+++ ("page_options", "phase", "trigger_state", "dispatched", "recorded"),+ [+ ({"trial_error": PlaywrightError("strict mode violation: 2 elements")}, "trigger", "not_attempted", 0, []),+ ({"click_error": PlaywrightError("element was detached")}, "trigger", "unknown", 1, [ActionType.CLICK]),+ ({"opens_chooser": False}, "chooser", "confirmed", 1, [ActionType.CLICK]),+ ({"set_files_error": PlaywrightError("chooser closed")}, "attachment", "confirmed", 1, [ActionType.CLICK]),+ ],+ ids=["trial-fails", "click-raises", "no-chooser", "set-files-fails"],+)+async def test_a_chooser_failure_names_its_phase_and_never_clicks_twice(+ authorized_file,+ monkeypatch: pytest.MonkeyPatch,+ page_options: dict[str, Exception | bool],+ phase: str,+ trigger_state: str,+ dispatched: int,+ recorded: list[ActionType],+) -> None:+ monkeypatch.setattr(code_block_authorized_files, "_CHOOSER_WAIT_SECONDS", 0.2)+ path, materialized = authorized_file+ raw_page = FakeChooserPage(**page_options)+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)++ started = time.monotonic()+ with pytest.raises(AuthorizedFileChooserError) as failure:+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")++ assert time.monotonic() - started < 2+ assert (failure.value.phase, failure.value.trigger_state) == (phase, trigger_state)+ assert failure.value.__cause__ is None+ assert "retained authorized bytes" not in str(failure.value)+ assert raw_page.dispatched_clicks == dispatched+ assert _recorded_types(recording_page) == recorded+ assert raw_page.chosen == [] and raw_page.input_files == []+ assert raw_page.live_listeners == 0+++async def test_a_timed_out_chooser_leaves_nothing_for_the_next_attach_to_collect(+ authorized_file, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(code_block_authorized_files, "_CHOOSER_WAIT_SECONDS", 0.2)+ path, materialized = authorized_file+ raw_page = FakeChooserPage(opens_chooser=False)+ first_page, first_attach = _bind_chooser_attach(raw_page, path, materialized)+ with pytest.raises(AuthorizedFileChooserError):+ await first_attach(first_page, str(path), "#choose-file", mode="file_chooser")+ assert raw_page.live_listeners == 0++ raw_page.opens_chooser = True+ second_page, second_attach = _bind_chooser_attach(raw_page, path, materialized)+ await second_attach(second_page, str(path), "#choose-file", mode="file_chooser")++ assert len(raw_page.chosen) == 1+ assert raw_page.dispatched_clicks == 2+ assert raw_page.live_listeners == 0+++async def test_cancelling_a_chooser_attach_propagates_and_removes_its_listener(authorized_file) -> None:+ path, materialized = authorized_file+ raw_page = FakeChooserPage(click_blocks=True)+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)++ task = asyncio.create_task(attach(recording_page, str(path), "#choose-file", mode="file_chooser"))+ while raw_page.dispatched_clicks == 0:+ await asyncio.sleep(0)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ assert raw_page.live_listeners == 0+ assert raw_page.dispatched_clicks == 1+ assert raw_page.chosen == []+++ ("set_files_stalls", "phase", "trigger_state", "set_files_calls"),+ [(False, "trigger", "unknown", 0), (True, "attachment", "confirmed", 1)],+ ids=["click", "set-files"],+)+async def test_a_chooser_call_cut_off_by_the_bound_stops_in_the_driver_too(+ authorized_file,+ monkeypatch: pytest.MonkeyPatch,+ set_files_stalls: bool,+ phase: str,+ trigger_state: str,+ set_files_calls: int,+) -> None:+ """A call whose caller is cancelled keeps running, so only its own timeout keeps it inside the bound."""+ monkeypatch.setattr(code_block_authorized_files, "_CHOOSER_WAIT_SECONDS", 0.2)+ path, materialized = authorized_file+ raw_page = FakeChooserPage(click_outlives_cancel=not set_files_stalls, set_files_outlives_cancel=set_files_stalls)+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)++ with pytest.raises(AuthorizedFileChooserError) as failure:+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")++ assert (failure.value.phase, failure.value.trigger_state) == (phase, trigger_state)+ assert [trial for _, trial in raw_page.clicks] == [True, False]+ assert len(raw_page.set_files_timeouts) == set_files_calls+ timeouts = raw_page.click_timeouts + raw_page.set_files_timeouts+ assert all(timeout is not None and 1 <= timeout <= 200 for timeout in timeouts)+ _, still_running = await asyncio.wait(raw_page.driver_calls, timeout=1)+ assert still_running == set()+++async def test_a_chooser_attach_leaves_an_agent_upload_listener_on_another_page_alone(authorized_file) -> None:+ path, materialized = authorized_file+ raw_page = FakeChooserPage()+ other_page = FakeChooserPage()+ other_chosen: list[FileChooserHandle] = []+ other_handler = other_chosen.append+ other_page.on("filechooser", other_handler)+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)++ with skyvern_context.scoped(SkyvernContext()) as context:+ pending = PendingFileChooserListener(+ page=other_page, # type: ignore[arg-type]+ file_paths=["/tmp/agent-upload.pdf"],+ handler=other_handler,+ )+ context.pending_file_chooser = pending+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")+ assert context.pending_file_chooser is pending+ assert other_page.listeners["filechooser"] == [other_handler]++ assert [files["buffer"] for files in raw_page.chosen] == [b"retained authorized bytes"]+++ ("file", "selector", "mode", "bind_overrides", "tamper"),+ [+ ("resume", "#choose-file", "chooser", {}, False),+ ("resume", "#choose-file", "File_Chooser", {}, False),+ ("raw-path", "#choose-file", "file_chooser", {}, False),+ ("https://invalid.example/cover.pdf", "#choose-file", "file_chooser", {}, False),+ ("resume", "", "file_chooser", {}, False),+ ("resume", "#choose-file", "file_chooser", {"workflow_run_id": "wr_other"}, False),+ ("resume", "#choose-file", "file_chooser", {"organization_id": "org_other"}, False),+ ("resume", "#choose-file", "file_chooser", {"max_bytes": 4}, False),+ ("resume", "#choose-file", "file_chooser", {}, True),+ ],+ ids=[+ "unknown-mode",+ "mode-is-exact",+ "raw-path",+ "unmaterialized",+ "empty-selector",+ "wrong-run",+ "wrong-org",+ "oversized",+ "tampered",+ ],+)+async def test_a_refused_chooser_attach_never_clicks_or_hands_over_bytes(+ authorized_file,+ tmp_path: Path,+ file: str,+ selector: str,+ mode: str,+ bind_overrides: dict[str, str | int],+ tamper: bool,+) -> None:+ path, materialized = authorized_file+ if tamper:+ path.write_bytes(b"x" * materialized.size)+ raw_path = tmp_path / "guessed.pdf"+ raw_path.write_bytes(b"not authorized")+ target = {"resume": str(path), "raw-path": str(raw_path)}.get(file, file)+ raw_page = FakeChooserPage()+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized, **bind_overrides)++ with pytest.raises(AuthorizedFileAccessError) as refusal:+ await attach(recording_page, target, selector, mode=mode)++ assert "retained authorized bytes" not in str(refusal.value)+ assert raw_page.clicks == []+ assert raw_page.chosen == [] and raw_page.input_files == []+ assert raw_page.live_listeners == 0+ assert _recorded_types(recording_page) == []+++async def test_a_chooser_attach_on_another_page_is_refused_before_any_click(authorized_file) -> None:+ path, materialized = authorized_file+ raw_page = FakeChooserPage()+ _, attach = _bind_chooser_attach(raw_page, path, materialized)++ with pytest.raises(AuthorizedFileAccessError, match="current CodeBlock page"):+ await attach(RecordingPage(FakeChooserPage()), str(path), "#choose-file", mode="file_chooser") # type: ignore[arg-type]++ assert raw_page.clicks == []+++async def test_a_chooser_attach_clears_an_agent_upload_listener_left_on_its_page(authorized_file) -> None:+ """An agent UPLOAD_FILE whose click opened no chooser leaves its listener armed; it must not fill this chooser."""+ path, materialized = authorized_file+ raw_page = FakeChooserPage()+ foreign: list[FileChooserHandle] = []+ foreign_handler = foreign.append+ raw_page.on("filechooser", foreign_handler)+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)++ with skyvern_context.scoped(SkyvernContext()) as context:+ context.pending_file_chooser = PendingFileChooserListener(+ page=raw_page, # type: ignore[arg-type]+ file_paths=["/tmp/agent-upload.pdf"],+ handler=foreign_handler,+ )+ receipt = await attach(recording_page, str(path), "#choose-file", mode="file_chooser")+ assert context.pending_file_chooser is None++ assert foreign == []+ assert receipt == {"filename": "resume.pdf", "size": len(b"retained authorized bytes")}+ assert [files["buffer"] for files in raw_page.chosen] == [b"retained authorized bytes"]+ assert raw_page.live_listeners == 0+++ ("remaining", "phase", "trigger_state", "dispatched"),+ [(1.4, "chooser", "confirmed", 1), (0.5, "trigger", "not_attempted", 0)],+ ids=["bound-clamped-to-the-deadline", "no-time-left-to-click"],+)+async def test_a_chooser_attach_returns_its_typed_failure_before_the_block_deadline(+ authorized_file, remaining: float, phase: str, trigger_state: str, dispatched: int+) -> None:+ path, materialized = authorized_file+ raw_page = FakeChooserPage(opens_chooser=False)+ deadline = time.monotonic() + remaining+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized, deadline=deadline)++ with pytest.raises(AuthorizedFileChooserError) as failure:+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")++ assert time.monotonic() < deadline+ assert (failure.value.phase, failure.value.trigger_state) == (phase, trigger_state)+ assert raw_page.dispatched_clicks == dispatched+ assert raw_page.live_listeners == 0+++def test_only_a_playwright_family_page_offers_chooser_mode() -> None:+ assert authorized_file_attach_modes(FakeChooserPage()) == {"input", "file_chooser"}+ assert authorized_file_attach_modes(FakeRawCDPChooserPage()) == {"input"}+ assert authorized_file_attach_modes(_RawPage()) == {"input"}+ assert pin_file_chooser(FakeRawCDPChooserPage()) is None+++async def test_a_page_without_chooser_mode_reports_it_unavailable_instead_of_filling_an_input(authorized_file) -> None:+ path, materialized = authorized_file+ raw_page = FakeRawCDPChooserPage()+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)++ with pytest.raises(AuthorizedFileChooserError) as failure:+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")++ assert str(failure.value) == AUTHORIZED_FILE_CHOOSER_UNAVAILABLE_ERROR+ assert (failure.value.phase, failure.value.trigger_state) == ("chooser", "not_attempted")+ assert raw_page.clicks == [] and raw_page.input_files == []+ assert await attach(recording_page, str(path), "#direct-file") == {+ "filename": "resume.pdf",+ "size": len(b"retained authorized bytes"),+ }+ assert len(raw_page.input_files) == 1+++async def test_an_inline_chooser_failure_after_the_click_is_not_healable(authorized_file) -> None:+ """The click may have fired, so self-heal must not re-run the block and click the trigger a second time."""+ path, materialized = authorized_file+ raw_page = FakeChooserPage(click_error=PlaywrightError("element was detached"))+ recording_page = RecordingPage(raw_page) # type: ignore[arg-type]+ now = datetime.now(UTC)+ block = CodeBlock(+ label="chooser_block",+ code='await attach_authorized_file(page, resume, "#choose-file", mode="file_chooser")',+ output_parameter=OutputParameter(+ parameter_type=ParameterType.OUTPUT,+ key="chooser_output",+ description="test output",+ output_parameter_id="op_chooser",+ workflow_id="w_test",+ created_at=now,+ modified_at=now,+ ),+ )+ run = block.generate_async_user_function(+ block.code,+ recording_page, # type: ignore[arg-type]+ {"resume": str(path)},+ workflow_run_id="wr_authorized",+ organization_id="org_authorized",+ authorized_file_materializations={"resume": materialized},+ )++ with pytest.raises(AuthorizedFileChooserError) as failure:+ await run()++ assert (failure.value.phase, failure.value.trigger_state) == ("trigger", "unknown")+ assert block._is_healable_page_failure(failure.value, recording_page) is False+ assert raw_page.dispatched_clicks == 1+ assert _recorded_types(recording_page) == [ActionType.CLICK]diff --git a/tests/unit/test_code_block_sandbox.py b/tests/unit/test_code_block_sandbox.pyindex 7ed20ad00..89ffee812 100644--- a/tests/unit/test_code_block_sandbox.py+++ b/tests/unit/test_code_block_sandbox.py@@ -3941,6 +3941,67 @@ class TestOpenPageHelperBinding:assert result.success is Falseassert "Failed on opened page: https://example.test/detail/3" in result.failure_reason+ @pytest.mark.asyncio+ async def test_a_failure_on_a_claimed_popup_names_that_popup_and_leaves_it_open(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ context = FakeSearchBrowserContext()+ timeout = PlaywrightTimeoutError("locator timed out")++ class PopupLocator:+ def __init__(self, page: FakeSearchPage) -> None:+ self.page = page++ async def click(self, **_kwargs: Any) -> None:+ raise timeout++ async def evaluate(self, _script: str, **_kwargs: Any) -> str:+ return '<div id="cover">'++ class PopupPage(FakeSearchPage):+ def locator(self, _selector: str, **_kwargs: Any) -> PopupLocator:+ return PopupLocator(self)++ popup = PopupPage(context=context)+ popup.url = "https://example.test/sign-in"++ @asynccontextmanager+ async def expect_popup(**_kwargs: Any) -> AsyncIterator[SimpleNamespace]:+ value: asyncio.Future[PopupPage] = asyncio.get_running_loop().create_future()+ value.set_result(popup)+ yield SimpleNamespace(value=value)++ root = SimpleNamespace(+ context=context, url="https://example.test/listing", is_closed=lambda: False, expect_popup=expect_popup+ )+ root.title = AsyncMock(return_value="Listing")+ browser_state = SimpleNamespace(+ browser_artifacts=BrowserArtifacts(),+ get_working_page=AsyncMock(return_value=root),+ list_valid_pages=AsyncMock(return_value=[root]),+ set_active_page=AsyncMock(),+ )+ monkeypatch.setattr(+ "skyvern.forge.sdk.workflow.models.block.app.AGENT_FUNCTION.validate_code_block", AsyncMock()+ )+ monkeypatch.setattr(CodeBlock, "get_or_create_browser_state", AsyncMock(return_value=browser_state))+ monkeypatch.setattr(CodeBlock, "get_workflow_run_context", lambda *args: FakeWorkflowRunContext(values={}))+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock())+ block = self._block()+ block.code = (+ "async with page.expect_popup() as popup_info:\n"+ " pass\n"+ "popup = await popup_info.value\n"+ "await popup.locator('#user').click()\n"+ )++ result = await block.execute(workflow_run_id="wrid_test", workflow_run_block_id="")++ assert result.success is False+ assert "Final URL: https://example.test/listing" in result.failure_reason+ assert "Failed on opened page: https://example.test/sign-in" in result.failure_reason+ assert popup.closed is False+@pytest.mark.asyncioasync def test_open_page_fails_closed_without_a_run_browser(self) -> None:with pytest.raises(RuntimeError, match="only supported while the run browser is open"):diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 715e000a9..8e31f721e 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -108,6 +108,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import ()from skyvern.forge.sdk.copilot.enforcement import (NUDGE_SENTINEL,+ RAW_SECRET_REPLY_WITHHELD_OBSERVATION,CopilotNonRetriableNavError,CopilotTotalTimeoutError,CopilotUnrecoverableToolError,@@ -127,6 +128,7 @@ from skyvern.forge.sdk.copilot.output_policy import OutputPolicyReason, OutputPofrom skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failurefrom skyvern.forge.sdk.copilot.request_policy import (_REDACTED_REFUSED_SECRET_TURN,+ RAW_SECRET_REFUSAL_SENTINEL,TRANSCRIPT_ANCHOR_CHAR_CAP,CompletionCriterion,RequestPolicy,@@ -7476,6 +7478,58 @@ class TestCopilotConfig:)assert blocked.result is blocked_result+ @staticmethod+ def _withheld_reply(*reason_codes: OutputPolicyReason) -> OutputGuardrailTripwireTriggered:+ guardrail_result = MagicMock()+ guardrail_result.output.output_info = {"allowed": False, "reason_codes": [code.value for code in reason_codes]}+ return OutputGuardrailTripwireTriggered(guardrail_result)++ @pytest.mark.asyncio+ async def test_reply_withheld_for_a_raw_secret_shape_gets_one_tool_less_reask(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ run = await _run_tool_bearing_empty_completion_turn(+ monkeypatch,+ _fake_run_result({"type": "REPLY", "user_response": "The page shows receipt FV-0F9080EE."}),+ attempts=(_EnforcementAttempt(error=self._withheld_reply(OutputPolicyReason.RAW_SECRET_LEAK)),),+ )++ [call] = run.drain_calls+ assert call.current_input == RAW_SECRET_REPLY_WITHHELD_OBSERVATION+ assert call.model_settings.tool_choice == "none"+ assert call.max_turns == 1+ assert run.result.user_response == "The page shows receipt FV-0F9080EE."++ @pytest.mark.asyncio+ async def test_reask_that_still_carries_a_raw_secret_keeps_the_refusal(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ run = await _run_tool_bearing_empty_completion_turn(+ monkeypatch,+ self._withheld_reply(OutputPolicyReason.RAW_SECRET_LEAK),+ attempts=(_EnforcementAttempt(error=self._withheld_reply(OutputPolicyReason.RAW_SECRET_LEAK)),),+ )++ assert len(run.drain_calls) == 1+ assert RAW_SECRET_REFUSAL_SENTINEL in run.result.user_response++ @pytest.mark.asyncio+ async def test_reply_withheld_for_another_reason_is_not_reasked(self, monkeypatch: pytest.MonkeyPatch) -> None:+ run = await _run_tool_bearing_empty_completion_turn(+ monkeypatch,+ _fake_run_result({"type": "REPLY", "user_response": "unused"}),+ attempts=(+ _EnforcementAttempt(+ error=self._withheld_reply(+ OutputPolicyReason.RAW_SECRET_LEAK, OutputPolicyReason.PERSISTENCE_STATE_MISMATCH+ )+ ),+ ),+ )++ assert run.drain_calls == []+ assert run.result.user_response != "unused"+@pytest.mark.asyncioasync def test_tool_bearing_empty_completion_final_reply_is_not_retried_for_untested_draft(self, monkeypatch: pytest.MonkeyPatchdiff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex d2b57225e..a850d1ad5 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -87,6 +87,7 @@ from skyvern.forge.sdk.copilot.secret_scrub import clear_session_scrub_values, rfrom skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modulefrom skyvern.forge.sdk.copilot.tools.composition_capture import store_post_run_page_evidencefrom skyvern.forge.sdk.copilot.tools.run_execution import (+ COPILOT_PROPOSAL_CHANGED_ERROR,_authored_literal_locator_selectors,_carry_unresolved_failure_into_result,_failed_block_code,@@ -122,6 +123,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (Workflow,WorkflowDefinition,WorkflowRunStatus,+ workflow_definition_sha256,)from skyvern.schemas.self_heal import HealEpisode, HealSkipReason, HealStatusfrom skyvern.schemas.workflows import BlockType@@ -136,8 +138,10 @@ from tests.unit.copilot_test_helpers import (count_record_and_send,failed_second_factor_run,handback_ctx,+ historical_code_version,inert_approval_workflow,install_get_run_results_harness,+ install_historical_run,install_run_blocks_harness,make_copilot_ctx,make_stub_html_artifact,@@ -2061,6 +2065,8 @@ async def test_failed_run_complete_fact_packet_reaches_ordinary_repair_input()run_workflow = SimpleNamespace(workflow_id="wf_run_snapshot",+ workflow_permanent_id=ctx.workflow_permanent_id,+ version=1,created_by=None,modified_at=datetime(2026, 4, 21, 12, 0),organization_id=ctx.organization_id,@@ -2077,12 +2083,14 @@ async def test_failed_run_complete_fact_packet_reaches_ordinary_repair_input()run = SimpleNamespace(workflow_run_id="wr_failed_complete_packet",+ workflow_id="wf_run_snapshot",workflow_permanent_id=ctx.workflow_permanent_id,browser_session_id="pbs_failed_complete_packet",status="failed",failure_reason="Code block failed.",created_at=datetime(2026, 4, 21, 12, 0),trigger_type=None,+ workflow_definition_sha256=None,)block = SimpleNamespace(workflow_run_block_id="wrb_failed_complete_packet",@@ -2209,7 +2217,8 @@ async def test_failed_run_complete_fact_packet_reaches_ordinary_repair_input(workflow_run_id="wr_failed_complete_packet",)assert nonassociated is not None- assert "execution_source" not in nonassociated["run"]+ assert nonassociated["run"]["execution_source"]["source_kind"] == "run_version"+ assert nonassociated["run"]["execution_source"]["workflow_id"] == "wf_run_snapshot"def test_unavailable_registered_output_rows_do_not_become_false_missing_output_facts() -> None:@@ -6048,7 +6057,7 @@ async def test_candidate_run_bind_uses_the_call_owned_source_and_token(assert result == {"ok": False,- "error": "The pending Copilot proposal changed before the test started; reload and try again.",+ "error": COPILOT_PROPOSAL_CHANGED_ERROR,}assert captured["expected_owner_turn_id"] == "turn-source"assert captured["expected_revision"] == 3@@ -8720,9 +8729,9 @@ async def test_read_run_reports_the_executed_versions_loop_inputs_not_the_currenmonkeypatch: pytest.MonkeyPatch,) -> None:monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))- executed = (- await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="w_source")- ).model_copy(update={"modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)})+ executed = (await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="wf-1")).model_copy(+ update={"workflow_permanent_id": "wpid-1", "modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)}+ )ctx = install_get_run_results_harness(monkeypatch,blocks=[@@ -8738,7 +8747,7 @@ async def test_read_run_reports_the_executed_versions_loop_inputs_not_the_currenpage = sanitize_tool_result_for_llm("get_run_results", project_run_results_page(result, {}))["data"]assert page["loop_inputs"]["workflow_run_id"] == "wr-1"- assert page["loop_inputs"]["workflow_id"] == "w_source"+ assert page["loop_inputs"]["workflow_id"] == "wf-1"[loop] = page["loop_inputs"]["loops"]assert (loop["loop_variable_reference"], loop["selected_input"], loop["loop_values_counts"]) == ("current_value",@@ -8773,8 +8782,12 @@ async def test_read_run_withholds_loop_inputs_when_the_workflow_row_changed_aftemonkeypatch: pytest.MonkeyPatch, workflow_yaml: str, created_by: str | None, expected_unavailable: bool) -> None:monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))- executed = (await inert_approval_workflow(workflow_yaml, workflow_id="w_source")).model_copy(- update={"created_by": created_by, "modified_at": HARNESS_RUN_CREATED_AT + timedelta(minutes=5)}+ executed = (await inert_approval_workflow(workflow_yaml, workflow_id="wf-1")).model_copy(+ update={+ "workflow_permanent_id": "wpid-1",+ "created_by": created_by,+ "modified_at": HARNESS_RUN_CREATED_AT + timedelta(minutes=5),+ })ctx = install_get_run_results_harness(monkeypatch,@@ -8874,8 +8887,10 @@ async def test_prior_run_packet_and_handoff_carry_the_notice_for_a_truncated_loo) -> None:monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))executed = (- await inert_approval_workflow(_parse_then_loop_yaml("current_value." + "a" * 400), workflow_id="w_source")- ).model_copy(update={"modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)})+ await inert_approval_workflow(_parse_then_loop_yaml("current_value." + "a" * 400), workflow_id="wf-1")+ ).model_copy(+ update={"workflow_permanent_id": "wpid-1", "modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)}+ )ctx = install_get_run_results_harness(monkeypatch,blocks=[@@ -8949,3 +8964,187 @@ def test_many_long_loop_facts_stay_inside_the_packet_budget(ai_fallback_rows: inassert all(len(loop.loop_variable_reference or "") <= 200 for loop in projected.run.loop_inputs.loops)assert all(loop.selected_input == "loop_variable_reference" for loop in projected.run.loop_inputs.loops)assert all(loop.loop_values_counts == [1_000_000] * 10 for loop in projected.run.loop_inputs.loops)+++_EXECUTED_A_CODE = 'greeting = "Hello " + legacy_applicant_name\nreturn {"greeting": greeting}\n'+_EDITABLE_B_CODE = 'display = greeting_prefix + " " + applicant_display_name\nreturn {"greeting": display}\n'+_EDITABLE_B_YAML = """title: Historical run source+workflow_definition:+ parameters:+ - parameter_type: workflow+ key: full_name+ workflow_parameter_type: string+ - parameter_type: workflow+ key: greeting_prefix+ workflow_parameter_type: string+ blocks:+ - block_type: code+ label: greet_applicant+ parameter_keys: [full_name, greeting_prefix]+ code: |+ display = greeting_prefix + " " + applicant_display_name+ return {"greeting": display}+"""+++def _editable_b_ctx() -> CopilotContext:+ return make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml=_EDITABLE_B_YAML)+++async def test_get_run_results_and_hydration_name_the_version_a_finished_run_executed(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ install_historical_run(+ monkeypatch, historical_code_version({"greet_applicant": _EXECUTED_A_CODE}), failed_label="greet_applicant"+ )+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args: None)+ ctx = _editable_b_ctx()++ raw = await tools_module.get_run_results_tool.on_invoke_tool(+ SimpleNamespace(context=ctx, tool_name="get_run_results"), json.dumps({"workflow_run_id": "wr-1"})+ )+ hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")++ tool_source = json.loads(raw)["data"]["execution_source"]+ assert hydrated is not None+ assert hydrated["run"]["execution_source"] == tool_source+ assert tool_source == {+ "source_kind": "run_version",+ "disposition": "available",+ "workflow_run_id": "wr-1",+ "workflow_id": "wf-1",+ "workflow_version": 3,+ "failed_block_label": "greet_applicant",+ "failed_statement": 'greeting = "Hello " + legacy_applicant_name',+ "declared_parameter_keys": ["applicant_name"],+ "block_code_sha256": {"greet_applicant": hashlib.sha256(_EXECUTED_A_CODE.encode()).hexdigest()},+ "current_draft_code_matches": {"greet_applicant": False},+ "failed_block_code": _EXECUTED_A_CODE,+ }+ executed_fields = json.dumps(tool_source)+ assert "applicant_display_name" not in executed_fields+ assert hashlib.sha256(_EDITABLE_B_CODE.encode()).hexdigest() not in executed_fields+++async def test_a_historical_read_takes_no_signature_or_unverified_label_from_the_current_draft(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ install_historical_run(+ monkeypatch, historical_code_version({"greet_applicant": _EXECUTED_A_CODE}), failed_label="greet_applicant"+ )+ ctx = _editable_b_ctx()+ ctx.last_unverified_block_labels = ["greet_applicant", "only_in_b"]+ ctx.unbound_required_parameter_keys = ["full_name"]+ ctx.runner_code_block_associations_by_label = {"greet_applicant": "assoc_b"}+ history_before = copy.deepcopy(ctx.recorded_build_test_outcome_history)++ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)+ outcome = run_execution_module._build_recorded_build_test_outcome(ctx, result, recorded_run_outcome=None)+ hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")+ draft_outcome = run_execution_module._build_recorded_build_test_outcome(+ ctx,+ {**result, "data": {key: value for key, value in result["data"].items() if key != "execution_source"}},+ recorded_run_outcome=None,+ )++ assert draft_outcome is not None and draft_outcome.authored_structure_signature+ assert outcome is not None+ assert outcome.workflow_run_id == "wr-1"+ assert outcome.authored_structure_signature is None+ assert outcome.block_shape_hashes == {}+ assert outcome.executed_block_associations == ()+ assert hydrated is not None+ assert [item for item in hydrated.get("unfinished_items", []) if item.get("kind") == "unverified_block"] == []+ assert ctx.recorded_build_test_outcome_history == history_before+ assert ctx.latest_recorded_build_test_outcome is None++[email protected]("definition_matches", [True, False], ids=["metadata-only-edit", "definition-rewritten"])+async def test_the_definition_recorded_at_dispatch_decides_whether_an_edited_row_is_what_ran(+ monkeypatch: pytest.MonkeyPatch, definition_matches: bool+) -> None:+ # A folder or title edit bumps modified_at past the run; only a definition change may withhold its source.+ version = historical_code_version(+ {"greet_applicant": _EXECUTED_A_CODE}, modified_at=HARNESS_RUN_CREATED_AT + timedelta(minutes=5)+ )+ recorded = historical_code_version({"greet_applicant": _EXECUTED_A_CODE if definition_matches else "pass\n"})+ install_historical_run(+ monkeypatch,+ version,+ failed_label="greet_applicant",+ run_definition_sha256=workflow_definition_sha256(recorded.workflow_definition),+ )+ observe_locators = AsyncMock(return_value=None)+ monkeypatch.setattr(run_execution_module, "_observe_authored_locators", observe_locators)++ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, _editable_b_ctx())++ source = result["data"]["execution_source"]+ if definition_matches:+ assert source["disposition"] == "available"+ assert source["failed_statement"] == 'greeting = "Hello " + legacy_applicant_name'+ assert observe_locators.await_args.kwargs["failed_block_code"] == _EXECUTED_A_CODE+ assert "requested_output_parameter_definitions" in result["data"]+ else:+ assert source == {+ "source_kind": "run_version",+ "disposition": "unavailable",+ "workflow_run_id": "wr-1",+ "workflow_id": "wf-1",+ "reason": "overwritten_after_run_start",+ }+ assert "requested_output_parameter_definitions" not in result["data"]+++ ("version", "reason"),+ [+ (None, "version_missing"),+ (historical_code_version({"greet_applicant": _EXECUTED_A_CODE}, workflow_id="wf-other"), "identity_mismatch"),+ (+ historical_code_version(+ {"greet_applicant": _EXECUTED_A_CODE}, modified_at=HARNESS_RUN_CREATED_AT + timedelta(seconds=1)+ ),+ "overwritten_after_run_start",+ ),+ (historical_code_version({"greet_applicant": _EXECUTED_A_CODE}), None),+ ],+)+async def test_an_unprovable_executed_version_is_unavailable_and_never_falls_back_to_the_draft(+ monkeypatch: pytest.MonkeyPatch, version: SimpleNamespace | None, reason: str | None+) -> None:+ install_historical_run(monkeypatch, version, failed_label="greet_applicant")+ observe_locators = AsyncMock(return_value=None)+ monkeypatch.setattr(run_execution_module, "_observe_authored_locators", observe_locators)+ ctx = _editable_b_ctx()++ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)+ outcome = run_execution_module._build_recorded_build_test_outcome(ctx, result, recorded_run_outcome=None)++ source = result["data"]["execution_source"]+ observed_code = observe_locators.await_args.kwargs["failed_block_code"] if observe_locators.await_args else None+ assert outcome is not None and outcome.authored_structure_signature is None+ assert "applicant_display_name" not in json.dumps(source)+ fetched_terminal_page = run_execution_module._fetch_dispatched_terminal_page_evidence.await_count > 0+ if reason is None:+ assert source["disposition"] == "available"+ assert source["failed_statement"] == 'greeting = "Hello " + legacy_applicant_name'+ assert observed_code == _EXECUTED_A_CODE+ assert "requested_output_parameter_definitions" in result["data"]+ assert fetched_terminal_page+ return+ # An unproven row may have lost the sensitive parameters or outputs the run had, so nothing is read from it.+ assert observed_code is None+ assert "requested_output_parameter_definitions" not in result["data"]+ assert not fetched_terminal_page+ assert source == {+ "source_kind": "run_version",+ "disposition": "unavailable",+ "workflow_run_id": "wr-1",+ "workflow_id": "wf-1",+ "reason": reason,+ }diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.pyindex 4056f00c9..5d2b76a0b 100644--- a/tests/unit/test_copilot_credential_pause.py+++ b/tests/unit/test_copilot_credential_pause.py@@ -20,9 +20,11 @@ Covers:from __future__ import annotationsimport asyncio+import ioimport json+import loggingimport time-from collections.abc import Awaitable, Callable+from collections.abc import Awaitable, Callable, Iteratorfrom copy import deepcopyfrom dataclasses import replacefrom datetime import datetime, timedelta, timezone@@ -32,12 +34,14 @@ from typing import Any, Literalfrom unittest.mock import AsyncMock, MagicMockimport pytest+import structlogfrom agents import RunConfigfrom agents.tool_context import ToolContext-from fastapi import HTTPException, status+from fastapi import BackgroundTasks, HTTPException, statusfrom structlog.testing import capture_logsfrom skyvern.config import settings+from skyvern.exceptions import HttpException as VaultHttpExceptionfrom skyvern.forge import appfrom skyvern.forge.sdk.cache.base import NoopLockfrom skyvern.forge.sdk.copilot import credential_pause as credential_pause_module@@ -56,11 +60,14 @@ from skyvern.forge.sdk.copilot.context import (StructuredContext,record_approved_credentials_in_global_llm_context,)+from skyvern.forge.sdk.copilot.credential_generation import SAFE_SYMBOLS, generate_registration_passwordfrom skyvern.forge.sdk.copilot.credential_pause import (CredentialPauseRejection,CredentialPauseResolution,+ _decode_active_pause,_encode_active_pause,await_pending_credential_pause,+ claim_credential_generation,credential_pause_active_key,credential_response_cache_key,encode_credential_response,@@ -96,20 +103,30 @@ from skyvern.forge.sdk.copilot.tools._shared import TOTAL_TIMEOUT_SECONDS, _copifrom skyvern.forge.sdk.copilot.tools.credential_fill import _request_credentialfrom skyvern.forge.sdk.copilot.tools.guardrails import _authority_tool_errorfrom skyvern.forge.sdk.copilot.turn_origin import TurnOrigin+from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.event_source_stream import EventSourceStream+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.forge_log import setup_logger+from skyvern.forge.sdk.routes import workflow_copilot as copilot_routesfrom skyvern.forge.sdk.routes.workflow_copilot import (WorkflowCopilotCredentialResponseRequest,+ workflow_copilot_credential_generate,workflow_copilot_credential_response,)from skyvern.forge.sdk.schemas.credentials import (+ CreateCredentialRequest,Credential,+ CredentialItem,CredentialType,CredentialVaultType,PasswordCredential,TotpType,)from skyvern.forge.sdk.schemas.workflow_copilot import (+ WorkflowCopilotCredentialGenerateRequest,+ WorkflowCopilotCredentialGenerateResult,WorkflowCopilotCredentialPauseResolvedUpdate,+ WorkflowCopilotCredentialRegistration,WorkflowCopilotCredentialRequiredUpdate,WorkflowCopilotStreamMessageType,)@@ -1718,6 +1735,22 @@ def test_connected_resume_stamps_nothing_when_the_ask_spans_several_origins() ->assert policy.live_page_admitted_urls == {}+async def test_generated_credential_connected_from_a_multi_origin_ask_gets_no_fill_grant(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _tool_ctx(monkeypatch)+ ctx.request_policy = _ask_origin_policy("https://portal.example.com/login", "https://other.example.com/login")+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.org_credentials_for_turn = []++ credential_pause_module._apply_connected_credential_to_policy(ctx, ctx.request_policy, _make_credential("cred_new"))+ grant, error = await tools_module.credential_fill._credential_fill_origin_grant(ctx, "cred_new")++ assert ctx.request_policy.live_page_admitted_urls == {}+ assert grant is None and error++def test_connected_resume_names_the_picked_credential_superseding_an_earlier_mention() -> None:"""The fill seam's which-credential check is set equality, so a card answer must replace anearlier prose mention, not join it."""@@ -2894,3 +2927,913 @@ async def test_a_run_derived_card_answered_by_signing_in_resumes_with_facts_and_assert ctx.credential_pause_outcome == "signed_in"assert credential_prompt_reason(ctx.request_policy, None) is Noneassert await maybe_credential_pause(ctx, _fake_result(), _make_stream(), config) is None+++_REGISTRATION = {+ "username": "[email protected]",+ "credential_name": "Portal test account",+ "password_length": 24,+ "charset": "alphanumeric_symbols",+}+++ ("length", "charset", "classes"),+ [+ (24, "alphanumeric_symbols", 4),+ (128, "alphanumeric_symbols", 4),+ (24, "alphanumeric", 3),+ (128, "alphanumeric", 3),+ ],+)+def test_registration_password_has_the_length_and_every_class_of_its_charset(+ length: int, charset: Literal["alphanumeric", "alphanumeric_symbols"], classes: int+) -> None:+ for _ in range(50):+ password = generate_registration_password(length, charset)+ present = [+ any(c.islower() for c in password),+ any(c.isupper() for c in password),+ any(c.isdigit() for c in password),+ any(c in SAFE_SYMBOLS for c in password),+ ]+ assert len(password) == length+ assert sum(present) == classes+ assert all(c.isalnum() or c in SAFE_SYMBOLS for c in password)++[email protected]("length", [23, 129])+def test_registration_password_refuses_a_length_outside_its_bounds(length: int) -> None:+ with pytest.raises(ValueError):+ generate_registration_password(length, "alphanumeric")+++class _RegistrationVault:+ """Stands in for the shared create path: records what it was asked to store and can fail or stall."""++ def __init__(+ self,+ credential: Credential,+ *,+ reject: bool = False,+ fail: bool = False,+ delay: float = 0.0,+ stored_password: str | None = None,+ read_delay: float = 0.0,+ read_fails: bool = False,+ vault_status: int | None = None,+ prepare_delay: float = 0.0,+ ):+ self.credential = credential+ self.prepare_delay = prepare_delay+ self.vault_status = vault_status+ self.reject = reject+ self.fail = fail+ self.delay = delay+ self.stored_password = stored_password+ self.read_delay = read_delay+ self.read_fails = read_fails+ self.prepared = 0+ self.passwords: list[str] = []+ self.stored = 0+ self.hooks_run: list[str] = []++ async def prepare(self, organization_id: str, data: CreateCredentialRequest) -> _RegistrationVault:+ self.prepared += 1+ await asyncio.sleep(self.prepare_delay)+ if self.reject:+ raise HTTPException(status_code=400, detail="No credential vault is configured")+ return self++ async def store(+ self,+ organization_id: str,+ data: CreateCredentialRequest,+ created_by: str | None,+ service: _RegistrationVault,+ tasks: BackgroundTasks,+ ) -> Credential:+ self.passwords.append(data.credential.password)+ await asyncio.sleep(self.delay)+ if self.fail:+ try:+ raise RuntimeError(f"vault rejected password '{data.credential.password}'")+ except RuntimeError:+ logging.getLogger("skyvern.test.vault").exception("vault create failed")+ raise HTTPException(status_code=502, detail=f"Custom credential service returned HTTP 500 for {data.name}")+ self.stored += 1+ tasks.add_task(self.hooks_run.append, self.credential.credential_id)+ return self.credential++ async def create_credential(+ self, organization_id: str, data: CreateCredentialRequest, created_by: str | None+ ) -> Credential:+ self.passwords.append(data.credential.password)+ assert self.vault_status is not None+ raise VaultHttpException(self.vault_status, "https://vault.example.test/credentials")++ async def get_credential_item(self, db_credential: Credential) -> CredentialItem:+ await asyncio.sleep(self.read_delay)+ password = self.stored_password or self.passwords[-1]+ if self.read_fails:+ raise RuntimeError(f"vault read returned password '{password}' in an unexpected shape")+ return CredentialItem(+ item_id=db_credential.item_id,+ name=db_credential.name,+ credential_type=CredentialType.PASSWORD,+ credential=PasswordCredential(username="[email protected]", password=password),+ )++ def install(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(copilot_routes, "prepare_credential_create", self.prepare)+ if self.vault_status is None:+ monkeypatch.setattr(copilot_routes, "store_new_credential", self.store)+++def _registration_database(+ monkeypatch: pytest.MonkeyPatch, credential: Credential, readback: list[Credential] | None = None+) -> SimpleNamespace:+ database = SimpleNamespace(+ credentials=SimpleNamespace(+ get_credentials_by_ids=AsyncMock(return_value=[credential]),+ get_credentials=AsyncMock(return_value=readback or []),+ ),+ workflow_params=SimpleNamespace(record_pending_copilot_turn_credential_resume=AsyncMock()),+ )+ monkeypatch.setattr(credential_pause_module.app, "DATABASE", database)+ return database+++def _seed_registration_pause(+ cache: _FakeCache, *, expires_at: datetime | None = None, registration: bool = True+) -> None:+ cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")] = _encode_active_pause(+ "tok-1",+ expires_at or datetime.now(timezone.utc) + timedelta(minutes=5),+ registration=WorkflowCopilotCredentialRegistration(**_REGISTRATION) if registration else None,+ )+++async def _generate(+ *, resume_token: str = "tok-1", organization_id: str = "org-1"+) -> WorkflowCopilotCredentialGenerateResult:+ with skyvern_context.scoped(SkyvernContext()):+ return await workflow_copilot_credential_generate(+ WorkflowCopilotCredentialGenerateRequest(+ turn_id="turn-1", workflow_copilot_chat_id="chat-1", resume_token=resume_token+ ),+ organization=SimpleNamespace(organization_id=organization_id), # type: ignore[arg-type]+ current_user_id="user-1",+ )+++def _registration_record(cache: _FakeCache) -> WorkflowCopilotCredentialRegistration | None:+ record = _decode_active_pause(cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")])+ assert record is not None and record.status == "pending"+ return record.registration+++async def test_registration_card_generates_one_credential_and_resumes_with_only_its_id(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ cache = _FakeCache()+ ctx = _tool_ctx(monkeypatch, cache)+ ctx.client_supports_credential_generation = True+ credential = _make_credential("cred_new", "Portal test account")+ database = _registration_database(monkeypatch, credential)+ vault = _RegistrationVault(credential)+ vault.install(monkeypatch)+ responses: list[WorkflowCopilotCredentialGenerateResult] = []++ async def answer_with_generate(+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,+ ) -> bool:+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:+ responses.append(await _generate(resume_token=frame.resume_token))+ return True++ ctx.stream.send.side_effect = answer_with_generate+ with capture_logs() as logs:+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)++ [card] = _sent_cards(ctx)+ assert card.reason == "credential_registration"+ assert card.login_page_urls == ["https://portal.example.com/login"]+ assert card.registration is not None+ assert (card.registration.username, card.registration.credential_name) == (+ "[email protected]",+ "Portal test account",+ )+ assert [response.result for response in responses] == ["connected"]+ assert result["status"] == "connected"+ assert result["credential_id"] == "cred_new"+ assert (result["generated"], result["site_effect"]) == (True, "none")+ assert "next" not in result and "data" not in result+ assert ctx.request_policy.live_page_admitted_urls == {"cred_new": "https://portal.example.com/login"}+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ grant, error = await tools_module.credential_fill._credential_fill_origin_grant(ctx, "cred_new")+ assert error is None and grant is not None+ assert tools_module.credential_fill._within_grant("https://portal.example.com/signup", grant)+ assert not tools_module.credential_fill._within_grant("https://elsewhere.example.com/signup", grant)+ assert vault.stored == 1+ await asyncio.sleep(0)+ assert vault.hooks_run == ["cred_new"]+ database.workflow_params.record_pending_copilot_turn_credential_resume.assert_awaited()+ [password] = vault.passwords+ surfaces = [+ json.dumps(result),+ *(call.args[0].model_dump_json() for call in ctx.stream.send.await_args_list),+ *(str(value) for value in cache.store.values()),+ *(response.model_dump_json() for response in responses),+ json.dumps(logs, default=str),+ ]+ assert all(password not in surface for surface in surfaces)++[email protected]("answer", ["generate", "pick"])+async def test_registration_card_during_origin_recovery_admits_its_generated_credential_without_site_evidence(+ monkeypatch: pytest.MonkeyPatch, answer: str+) -> None:+ cache = _FakeCache()+ ctx = _recovery_ctx(monkeypatch, cache)+ ctx.client_supports_credential_generation = True+ generated = _make_credential("cred_new", "Portal test account")+ picked = _make_credential("cred_blank")+ database = _registration_database(monkeypatch, generated if answer == "generate" else picked)+ database.credentials.get_credential = AsyncMock(return_value=None)+ vault = _RegistrationVault(generated)+ vault.install(monkeypatch)++ async def answer_card(+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,+ ) -> bool:+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:+ if answer == "generate":+ await _generate(resume_token=frame.resume_token)+ else:+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(+ "connected", "cred_blank", frame.resume_token+ )+ return True++ ctx.stream.send.side_effect = answer_card+ result = await _call_ask_tool(ctx, login_page_url=f"{_IDP_ORIGIN}/login", registration=_REGISTRATION)++ [card] = _sent_cards(ctx)+ assert card.registration is not None+ if answer == "generate":+ assert (result["status"], result["credential_id"]) == ("connected", "cred_new")+ assert ctx.credential_pause_outcome == "connected"+ assert ctx.request_policy.live_page_admitted_urls["cred_new"] == f"{_IDP_ORIGIN}/login"+ assert ctx.credential_origin_recovery is None+ else:+ assert vault.passwords == []+ assert "cred_blank" not in ctx.request_policy.live_page_admitted_urls+ assert ctx.credential_pause_outcome == "not_admitted"+ _assert_declined(ctx, result, "connected_other_site")+++async def test_registration_ask_on_an_origin_recovery_handback_after_a_generate_raises_no_card(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ cache = _FakeCache()+ ctx = _tool_ctx(monkeypatch, cache)+ ctx.client_supports_credential_generation = True+ generated = _make_credential("cred_new", "Portal test account")+ _registration_database(monkeypatch, generated)+ vault = _RegistrationVault(generated)+ vault.install(monkeypatch)++ async def answer_with_generate(+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,+ ) -> bool:+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:+ await _generate(resume_token=frame.resume_token)+ return True++ ctx.stream.send.side_effect = answer_with_generate+ first = await _call_ask_tool(ctx, registration=_REGISTRATION)+ assert first["status"] == "connected"+ ctx.credential_origin_recovery = _PENDING++ handback = await _call_ask_tool(ctx, login_page_url=f"{_IDP_ORIGIN}/login", registration=_REGISTRATION)++ assert handback["status"] == "already_generated"+ assert len(_sent_cards(ctx)) == 1+ assert (vault.stored, len(vault.passwords)) == (1, 1)+ assert ctx.credential_origin_recovery == _PENDING++[email protected]("reask", ["origin_recovery_handback", "run_budget_reset"])[email protected]("outcome", ["unknown", "rejected"])+async def test_registration_reask_after_an_unconfirmed_generate_never_creates_a_second_credential(+ monkeypatch: pytest.MonkeyPatch, outcome: str, reask: str+) -> None:+ cache = _FakeCache()+ ctx = _tool_ctx(monkeypatch, cache)+ ctx.client_supports_credential_generation = True+ _registration_database(monkeypatch, _make_credential("cred_new", "Portal test account"))+ vault = _RegistrationVault(+ _make_credential("cred_new", "Portal test account"), reject=outcome == "rejected", fail=outcome == "unknown"+ )+ vault.install(monkeypatch)+ generated: list[str] = []++ async def generate_then_skip(+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,+ ) -> bool:+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:+ generated.append((await _generate(resume_token=frame.resume_token)).result)+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id=frame.turn_id,+ resume_token=frame.resume_token,+ action="skip",+ credential_id=None,+ )+ return True++ ctx.stream.send.side_effect = generate_then_skip+ first = await _call_ask_tool(ctx, registration=_REGISTRATION)+ assert (first["status"], generated) == ("skipped", [outcome])+ assert ctx.credential_pause_reaskable_by_run is True+ login_page_url = "https://portal.example.com/login"+ if reask == "origin_recovery_handback":+ ctx.credential_origin_recovery = _PENDING+ login_page_url = f"{_IDP_ORIGIN}/login"+ else:+ ctx.credential_pause_used = False++ second = await _call_ask_tool(ctx, login_page_url=login_page_url, registration=_REGISTRATION)++ if outcome == "unknown":+ assert second["status"] == "already_generated"+ assert len(_sent_cards(ctx)) == 1+ assert (vault.prepared, len(vault.passwords), generated) == (1, 1, ["unknown"])+ else:+ assert len(_sent_cards(ctx)) == 2+ assert (vault.prepared, vault.passwords, generated) == (2, [], ["rejected", "rejected"])+++async def test_registration_card_skipped_creates_no_credential_and_cannot_generate_afterwards(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ cache = _FakeCache()+ ctx = _tool_ctx(monkeypatch, cache)+ ctx.client_supports_credential_generation = True+ _registration_database(monkeypatch, _make_credential())+ vault = _RegistrationVault(_make_credential())+ vault.install(monkeypatch)+ ctx.stream.send.side_effect = _answer_each_card(cache, [("skip", None)])++ result = await _call_ask_tool(ctx, registration=_REGISTRATION)++ assert result["status"] == "skipped"+ with pytest.raises(HTTPException) as late:+ await _generate()+ assert late.value.status_code == status.HTTP_409_CONFLICT+ assert (vault.prepared, vault.passwords) == (0, [])+++async def test_registration_card_answered_with_a_saved_login_does_not_report_it_generated(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ cache = _FakeCache()+ ctx = _tool_ctx(monkeypatch, cache)+ ctx.client_supports_credential_generation = True+ _registration_database(monkeypatch, _make_credential("cred_saved", "Saved login"))+ vault = _RegistrationVault(_make_credential())+ vault.install(monkeypatch)+ ctx.stream.send.side_effect = _answer_each_card(cache, [("connected", "cred_saved")])++ result = await _call_ask_tool(ctx, registration=_REGISTRATION)++ assert (result["status"], result["credential_id"]) == ("connected", "cred_saved")+ assert "generated" not in result and "site_effect" not in result+ assert (vault.prepared, vault.passwords) == (0, [])+++ ("registration", "expected"),+ [+ ({**_REGISTRATION, "password_length": 23}, "unsupported_constraints"),+ ({**_REGISTRATION, "password_length": 129}, "unsupported_constraints"),+ ({**_REGISTRATION, "username": " "}, None),+ ],+)+async def test_registration_ask_outside_the_typed_bounds_raises_no_card(+ monkeypatch: pytest.MonkeyPatch, registration: dict[str, Any], expected: str | None+) -> None:+ ctx = _tool_ctx(monkeypatch)+ ctx.client_supports_credential_generation = True++ result = await _call_ask_tool(ctx, registration=registration)++ assert result.get("status") == expected+ assert "fallback" not in result and "next" not in result+ assert _sent_cards(ctx) == []+ assert ctx.credential_pause_used is False+++async def test_registration_ask_is_unavailable_when_the_client_cannot_generate(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _tool_ctx(monkeypatch)+ ctx.client_supports_credential_generation = False++ result = await _call_ask_tool(ctx, registration=_REGISTRATION)++ assert result["status"] == "unavailable"+ assert "fallback" not in result and "next" not in result+ assert _sent_cards(ctx) == []++[email protected]("conflict", ["credential_id", "raw_secret"])+async def test_registration_ask_is_refused_beside_a_credential_id_or_on_a_secret_turn(+ monkeypatch: pytest.MonkeyPatch, conflict: str+) -> None:+ ctx = _tool_ctx(monkeypatch)+ ctx.client_supports_credential_generation = True+ extra: dict[str, Any] = {}+ if conflict == "credential_id":+ extra["credential_id"] = "cred_1"+ else:+ ctx.request_policy = _redacted_secret_policy("https://portal.example.com/login")++ result = await _call_ask_tool(ctx, registration=_REGISTRATION, **extra)++ assert result["ok"] is False+ assert _sent_cards(ctx) == []+++def _readback_credential(credential_id: str, **update: Any) -> Credential:+ return _make_credential(credential_id, "Portal test account").model_copy(+ update={+ "username": "[email protected]",+ "created_by": "user-1",+ "created_at": datetime.now(timezone.utc) + timedelta(seconds=5),+ **update,+ }+ )++[email protected]("failure", ["rejected", "slow_prepare", "vault_error", "timeout"])+async def test_registration_generate_failure_with_no_readback_match_reopens_the_card_once(+ monkeypatch: pytest.MonkeyPatch, failure: str+) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ _seed_registration_pause(cache)+ late = _readback_credential("cred_late")+ stale = _readback_credential("cred_stale", created_at=datetime.now(timezone.utc) - timedelta(hours=1))+ other_user = _readback_credential("cred_other", created_by="user-2")+ database = _registration_database(monkeypatch, late, readback=[other_user, stale])+ vault = _RegistrationVault(+ late,+ reject=failure == "rejected",+ fail=failure == "vault_error",+ delay=0.2 if failure == "timeout" else 0,+ prepare_delay=5 if failure == "slow_prepare" else 0,+ )+ vault.install(monkeypatch)+ monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 1)+ monkeypatch.setattr(copilot_routes, "_CLAIM_FINISH_MARGIN_SECONDS", 0.5)+ monkeypatch.setattr(copilot_routes, "_READBACK_RESERVE_SECONDS", 0.5)++ with capture_logs() as logs:+ started = time.monotonic()+ result = await _generate()+ assert time.monotonic() - started < 2+ expected = "rejected" if failure in ("rejected", "slow_prepare") else "unknown"+ assert result.result == expected+ assert result.credential_id is None+ if expected == "rejected":+ database.credentials.get_credentials.assert_not_awaited()+ registration = _registration_record(cache)+ assert registration is not None+ assert (registration.attempted, registration.outcome) == (True, expected)+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store+ database.workflow_params.record_pending_copilot_turn_credential_resume.assert_not_awaited()+ with pytest.raises(HTTPException) as again:+ await _generate()+ assert again.value.status_code == status.HTTP_409_CONFLICT+ assert len(vault.passwords) == (0 if expected == "rejected" else 1)+ await asyncio.sleep(0.3)+ assert vault.stored == (1 if failure == "timeout" else 0)+ assert vault.hooks_run == (["cred_late"] if failure == "timeout" else [])+ late_creates = [log["credential_id"] for log in logs if log["event"] == "copilot_credential_generation_late_create"]+ assert late_creates == (["cred_late"] if failure == "timeout" else [])+ for password in vault.passwords:+ assert password not in json.dumps(logs, default=str)+ assert password not in result.model_dump_json()+ assert password not in str(cache.store)++[email protected](("create_seconds", "expected_id"), [(0.2, "cred_new"), (1.5, None)])+async def test_registration_slow_create_connects_only_when_it_lands_before_the_wait_cutoff(+ monkeypatch: pytest.MonkeyPatch, create_seconds: float, expected_id: str | None+) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 2)+ monkeypatch.setattr(copilot_routes, "_CLAIM_FINISH_MARGIN_SECONDS", 0.5)+ monkeypatch.setattr(copilot_routes, "_READBACK_RESERVE_SECONDS", 0.5)+ _seed_registration_pause(cache)+ credential = _make_credential("cred_new", "Portal test account")+ database = _registration_database(monkeypatch, credential)+ vault = _RegistrationVault(credential, delay=create_seconds)+ vault.install(monkeypatch)++ result = await _generate()+ with pytest.raises(HTTPException) as again:+ await _generate()++ assert again.value.status_code == status.HTTP_409_CONFLICT+ assert len(vault.passwords) == 1+ if expected_id is not None:+ assert (result.result, result.credential_id) == ("connected", expected_id)+ database.credentials.get_credentials.assert_not_awaited()+ else:+ assert (result.result, result.credential_id) == ("unknown", None)+ database.credentials.get_credentials.assert_awaited_once()+ registration = _registration_record(cache)+ assert registration is not None and registration.outcome == "unknown"+ await asyncio.sleep(create_seconds)+ assert vault.stored == 1++[email protected](("vault_status", "expected"), [(422, "rejected"), (500, "unknown")])+async def test_registration_vault_refusal_is_rejected_only_for_a_4xx(+ monkeypatch: pytest.MonkeyPatch, vault_status: int, expected: str+) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ _seed_registration_pause(cache)+ credential = _readback_credential("cred_unrelated", created_by="user-2")+ database = _registration_database(monkeypatch, credential, readback=[credential])+ vault = _RegistrationVault(credential, vault_status=vault_status)+ vault.install(monkeypatch)++ with capture_logs() as logs:+ result = await _generate()+ with pytest.raises(HTTPException) as again:+ await _generate()++ assert (result.result, result.credential_id) == (expected, None)+ assert again.value.status_code == status.HTTP_409_CONFLICT+ assert len(vault.passwords) == 1+ if expected == "rejected":+ database.credentials.get_credentials.assert_not_awaited()+ else:+ database.credentials.get_credentials.assert_awaited()+ registration = _registration_record(cache)+ assert registration is not None and registration.outcome == expected+ password = vault.passwords[0]+ assert password not in json.dumps(logs, default=str)+ assert password not in result.model_dump_json()+ assert password not in str(cache.store)++[email protected]("readback", ["one", "two", "full_page", "other_password", "slow_read"])+async def test_registration_unknown_save_connects_only_a_single_readback_match(+ monkeypatch: pytest.MonkeyPatch, readback: str+) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 1)+ monkeypatch.setattr(copilot_routes, "_CLAIM_FINISH_MARGIN_SECONDS", 0.5)+ _seed_registration_pause(cache)+ match = _readback_credential("cred_match")+ rows = {+ "two": [match, _readback_credential("cred_twin")],+ "full_page": [match, *(_readback_credential(f"cred_{i}", username=f"u{i}@example.com") for i in range(99))],+ }.get(readback, [match])+ database = _registration_database(monkeypatch, match, readback=rows)+ vault = _RegistrationVault(+ match,+ fail=True,+ stored_password="Another-create's-password-1" if readback == "other_password" else None,+ read_delay=30 if readback == "slow_read" else 0,+ )+ vault.install(monkeypatch)++ started = time.monotonic()+ result = await _generate()++ assert time.monotonic() - started < 5++ assert len(vault.passwords) == 1+ with pytest.raises(HTTPException) as again:+ await _generate()+ assert again.value.status_code == status.HTTP_409_CONFLICT+ assert len(vault.passwords) == 1+ if readback == "one":+ assert (result.result, result.credential_id) == ("connected", "cred_match")+ response = json.loads(cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")])+ assert (response["action"], response["credential_id"]) == ("connected", "cred_match")+ database.workflow_params.record_pending_copilot_turn_credential_resume.assert_awaited()+ else:+ assert (result.result, result.credential_id) == ("unknown", None)+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store+ registration = _registration_record(cache)+ assert registration is not None and registration.outcome == "unknown"+++ ("case", "expected_status"),+ [+ ("bad_token", status.HTTP_403_FORBIDDEN),+ ("expired", status.HTTP_410_GONE),+ ("consumed", status.HTTP_409_CONFLICT),+ ("other_org", status.HTTP_404_NOT_FOUND),+ ("not_a_registration_card", status.HTTP_409_CONFLICT),+ ],+)+async def test_registration_generate_refuses_before_any_vault_call(+ monkeypatch: pytest.MonkeyPatch, case: str, expected_status: int+) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ _seed_registration_pause(+ cache,+ expires_at=datetime.now(timezone.utc) - timedelta(seconds=1) if case == "expired" else None,+ registration=case != "not_a_registration_card",+ )+ if case == "consumed":+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-1",+ resume_token="tok-1",+ action="skip",+ credential_id=None,+ )+ _registration_database(monkeypatch, _make_credential())+ vault = _RegistrationVault(_make_credential())+ vault.install(monkeypatch)++ with pytest.raises(HTTPException) as refused:+ await _generate(+ resume_token="tok-wrong" if case == "bad_token" else "tok-1",+ organization_id="org-2" if case == "other_org" else "org-1",+ )++ assert refused.value.status_code == expected_status+ assert (vault.prepared, vault.passwords) == (0, [])+++async def test_registration_created_after_the_pause_ended_is_reported_but_not_connected(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ _seed_registration_pause(cache)+ created = _make_credential("cred_late", "Portal test account")+ _registration_database(monkeypatch, created)+ vault = _RegistrationVault(created)+ vault.install(monkeypatch)+ store = vault.store++ async def store_while_the_waiter_gives_up(+ organization_id: str,+ data: CreateCredentialRequest,+ created_by: str | None,+ service: str,+ tasks: BackgroundTasks,+ ) -> Credential:+ cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")] = _encode_active_pause(+ "tok-1", datetime.now(timezone.utc), consumed=True+ )+ return await store(organization_id, data, created_by, service, tasks)++ monkeypatch.setattr(copilot_routes, "store_new_credential", store_while_the_waiter_gives_up)++ result = await _generate()++ assert result.result == "created_not_connected"+ assert result.credential_id == "cred_late"+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store+++async def test_registration_created_but_unrecorded_on_the_pause_is_reported_with_its_id(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ _seed_registration_pause(cache)+ created = _make_credential("cred_saved", "Portal test account")+ _registration_database(monkeypatch, created)+ vault = _RegistrationVault(created)+ vault.install(monkeypatch)+ response_key = credential_response_cache_key("org-1", "chat-1", "turn-1")+ cache_set = cache.set++ async def set_failing_on_the_response(key: str, value: Any, ex: Any = None) -> None:+ if key == response_key:+ raise ConnectionError("cache unavailable")+ await cache_set(key, value, ex)++ monkeypatch.setattr(cache, "set", set_failing_on_the_response)++ result = await _generate()++ assert (result.result, result.credential_id) == ("created_not_connected", "cred_saved")+ assert len(vault.passwords) == 1++[email protected]("failure", ["rejected", "vault_error"])+async def test_registration_card_reopened_after_its_deadline_passed_mid_save_is_still_answerable(+ monkeypatch: pytest.MonkeyPatch, failure: str+) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ _seed_registration_pause(cache, expires_at=datetime.now(timezone.utc) + timedelta(milliseconds=50))+ _registration_database(monkeypatch, _make_credential())+ vault = _RegistrationVault(+ _make_credential(), reject=failure == "rejected", fail=failure == "vault_error", delay=0.1+ )+ vault.install(monkeypatch)++ result = await _generate()++ record = _decode_active_pause(cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")])+ assert record is not None and record.status == "pending"+ assert result.expires_at == record.expires_at+ assert record.expires_at > datetime.now(timezone.utc) + timedelta(+ seconds=settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS - 5+ )+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-1",+ resume_token="tok-1",+ action="skip",+ credential_id=None,+ )+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") in cache.store+++async def test_registration_ask_with_a_charset_outside_the_enum_returns_an_error_the_model_sees(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _tool_ctx(monkeypatch)+ ctx.client_supports_credential_generation = True++ result = await _call_ask_tool(ctx, registration={**_REGISTRATION, "charset": "letters_and_spaces"})++ assert result["ok"] is False+ assert "registration.charset" in result["error"]+ assert _sent_cards(ctx) == []+ assert ctx.credential_pause_used is False++[email protected](params=[True, False], ids=["json", "console"])+def rendered_logs(monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest) -> Iterator[io.StringIO]:+ monkeypatch.setattr(settings, "JSON_LOGGING", request.param)+ root = logging.getLogger()+ saved_config = structlog.get_config()+ saved_handlers, saved_level = root.handlers[:], root.level+ stream = io.StringIO()+ try:+ setup_logger()+ handler = root.handlers[0]+ assert isinstance(handler, logging.StreamHandler)+ handler.setStream(stream)+ yield stream+ finally:+ structlog.configure(**saved_config)+ root.handlers[:] = saved_handlers+ root.setLevel(saved_level)+++async def test_registration_password_quoted_by_a_vault_error_is_scrubbed_from_rendered_logs_and_tracebacks(+ monkeypatch: pytest.MonkeyPatch, rendered_logs: io.StringIO+) -> None:+ cache = _FakeCache()+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)+ _seed_registration_pause(cache)+ match = _readback_credential("cred_match")+ _registration_database(monkeypatch, match, readback=[match])+ vault = _RegistrationVault(match, fail=True, read_fails=True)+ vault.install(monkeypatch)++ result = await _generate()++ [password] = vault.passwords+ rendered = rendered_logs.getvalue()+ assert result.result == "unknown"+ assert "vault create failed" in rendered+ assert "copilot_credential_generation_readback_failed" in rendered+ assert "in an unexpected shape" in rendered+ assert "Traceback" in rendered+ assert password not in rendered++[email protected]("answer", ["skip", "timeout"])+async def test_registration_card_reopened_after_an_unconfirmed_save_reports_it_when_skipped_or_unanswered(+ monkeypatch: pytest.MonkeyPatch, answer: str+) -> None:+ cache = _FakeCache()+ ctx = _tool_ctx(monkeypatch, cache)+ ctx.client_supports_credential_generation = True+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=1)+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS", 1)+ _registration_database(monkeypatch, _make_credential())+ vault = _RegistrationVault(_make_credential(), fail=True)+ vault.install(monkeypatch)++ async def generate_then_answer(+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,+ ) -> bool:+ if frame.type != WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:+ return True+ assert (await _generate(resume_token=frame.resume_token)).result == "unknown"+ if answer == "skip":+ await resolve_credential_pause(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-1",+ resume_token=frame.resume_token,+ action="skip",+ credential_id=None,+ )+ return True++ ctx.stream.send.side_effect = generate_then_answer+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)++ assert result["status"] == ("skipped" if answer == "skip" else "unanswered")+ assert result["registration_outcome"] == "unknown"+ assert "next" not in result+ assert len(vault.passwords) == 1+ assert vault.passwords[0] not in json.dumps(result)+++async def test_registration_card_that_ran_out_mid_create_reports_unknown(monkeypatch: pytest.MonkeyPatch) -> None:+ cache = _FakeCache()+ ctx = _tool_ctx(monkeypatch, cache)+ ctx.client_supports_credential_generation = True+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=1)+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS", 1)+ monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 1)++ async def claim_and_abandon(+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,+ ) -> bool:+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:+ await claim_credential_generation(+ cache,+ organization_id="org-1",+ workflow_copilot_chat_id="chat-1",+ turn_id="turn-1",+ resume_token=frame.resume_token,+ )+ return True++ ctx.stream.send.side_effect = claim_and_abandon+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)++ assert result["status"] == "unanswered"+ assert result["registration_outcome"] == "unknown"+++async def test_registration_outcome_from_an_earlier_ask_is_not_reported_on_the_next(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _tool_ctx(monkeypatch, _FakeCache())+ ctx.client_supports_credential_generation = True+ ctx.credential_registration_outcome = "rejected"+ ctx.client_supports_credential_pause_recovery = False+ ctx.stream.is_disconnected.return_value = True++ result = await _call_ask_tool(ctx, registration=_REGISTRATION)++ assert result["status"] == "unanswered"+ assert "registration_outcome" not in resultdiff --git a/tests/unit/test_copilot_credential_pause_reload.py b/tests/unit/test_copilot_credential_pause_reload.pyindex ac616f3b0..b2e428790 100644--- a/tests/unit/test_copilot_credential_pause_reload.py+++ b/tests/unit/test_copilot_credential_pause_reload.py@@ -12,10 +12,19 @@ from skyvern.forge.sdk.copilot import credential_pause as pausesfrom skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.request_policy import RequestPolicy, _ground_user_provided_sitesfrom skyvern.forge.sdk.copilot.tools.credential_fill import _request_credential+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.forge.sdk.routes import workflow_copilot as routesfrom skyvern.forge.sdk.schemas.browser_profiles import BrowserProfile+from skyvern.forge.sdk.schemas.workflow_copilot import CredentialRegistrationfrom tests.unit.test_copilot_ask_user import setup_question_chat-from tests.unit.test_copilot_credential_pause import _FakeCache, _make_credential, _make_stream+from tests.unit.test_copilot_credential_pause import (+ _REGISTRATION,+ _FakeCache,+ _make_credential,+ _make_stream,+ _RegistrationVault,+)@pytest.mark.asyncio@@ -406,3 +415,79 @@ async def test_a_done_whose_pause_ended_while_saving_discards_the_profile(sqlite)response_key = pauses.credential_response_cache_key("org", card.workflow_copilot_chat_id, card.turn_id)assert response_key not in cache.store+++async def test_registration_card_survives_a_reload_with_its_one_generate_attempt_recorded(sqlite_engine, monkeypatch):+ _, client, ctx, _ = await setup_question_chat(sqlite_engine, monkeypatch)+ monkeypatch.setattr(app, "CACHE", _FakeCache())+ monkeypatch.setattr(pauses, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)+ monkeypatch.setattr(routes, "RECONCILE_ABANDON_AFTER_SECONDS", -1)+ monkeypatch.setattr(app.DATABASE, "credentials", SimpleNamespace(get_credentials=AsyncMock(return_value=[])))+ emitted = _sign_in_card_context(ctx, recovery=True)+ ctx.client_supports_credential_generation = True+ recovery_token = "a" * 64+ vault = _RegistrationVault(_make_credential(), fail=True)+ vault.install(monkeypatch)+ recovered_while_saving: list[list] = []+ fail_store = vault.store++ async def store_and_reload(*args):+ recovered_while_saving.append(+ await pauses.pending_credential_requests("org", ctx.workflow_copilot_chat_id, ["turn"], recovery_token)+ )+ return await fail_store(*args)++ monkeypatch.setattr(routes, "store_new_credential", store_and_reload)+ api = client._transport.app+ api.dependency_overrides[routes.org_auth_service.get_current_org_for_credential_routes] = api.dependency_overrides[+ routes.org_auth_service.get_current_org+ ]+ api.dependency_overrides[routes.org_auth_service.get_current_user_id_or_none] = lambda: "user-1"+ api.add_api_route("/credential-generate", routes.workflow_copilot_credential_generate, methods=["POST"])++ async def history() -> list[dict]:+ response = await client.get(+ "/history",+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},+ )+ return response.json()["pending_credential_requests"]++ async with client:+ invocation = asyncio.create_task(+ _request_credential(+ "https://portal.example.com/signup",+ "Create the test account",+ ctx,+ registration=CredentialRegistration(**_REGISTRATION),+ )+ )+ try:+ await asyncio.wait_for(emitted.wait(), 1)+ [raised] = await history()+ assert raised["reason"] == "credential_registration"+ assert raised["registration"]["credential_name"] == "Portal test account"+ assert raised["registration"]["attempted"] is False+ body = {+ "workflow_copilot_chat_id": raised["workflow_copilot_chat_id"],+ "turn_id": raised["turn_id"],+ "resume_token": raised["resume_token"],+ }++ with skyvern_context.scoped(SkyvernContext()):+ generated = await client.post("/credential-generate", json=body)++ assert generated.json()["result"] == "unknown"+ [[saving]] = recovered_while_saving+ assert saving.registration is not None and saving.registration.attempted is True+ [reopened] = await history()+ assert reopened["registration"]["attempted"] is True+ assert reopened["registration"]["outcome"] == "unknown"+ with skyvern_context.scoped(SkyvernContext()):+ assert (await client.post("/credential-generate", json=body)).status_code == 409+ assert len(vault.passwords) == 1+ assert not invocation.done()+ finally:+ invocation.cancel()+ await asyncio.gather(invocation, return_exceptions=True)diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex b608f04c0..797f6565b 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -343,7 +343,7 @@ def test_non_paused_watchdog_exit_still_classifies_as_a_failed_run() -> None:source_tool="run_blocks_and_collect_debug",result={"ok": False,- "error": "The run exceeded the 600s absolute ceiling while still showing progress.",+ "error": "The run did not reach a terminal status within the 600s absolute ceiling.","data": {"workflow_run_id": "wr_ceiling","overall_status": "running",@@ -3474,6 +3474,9 @@ async def test_completed_missing_output_complete_fact_packet_reaches_ordinary_remodified_at=now,)run_workflow = SimpleNamespace(+ workflow_id="wf_run_snapshot",+ workflow_permanent_id=ctx.workflow_permanent_id,+ version=1,organization_id=ctx.organization_id,created_by=None,modified_at=datetime(2026, 4, 21, 12, 0),@@ -3484,12 +3487,14 @@ async def test_completed_missing_output_complete_fact_packet_reaches_ordinary_re)run = SimpleNamespace(workflow_run_id="wr_completed_missing_output",+ workflow_id="wf_run_snapshot",workflow_permanent_id=ctx.workflow_permanent_id,browser_session_id="pbs_completed_missing_output",status="completed",failure_reason=None,created_at=datetime(2026, 4, 21, 12, 0),trigger_type=None,+ workflow_definition_sha256=None,)block = WorkflowRunBlock(workflow_run_block_id="wrb_completed_missing_output",diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.pyindex 15fccdd47..b961281df 100644--- a/tests/unit/test_copilot_enforcement_pruning.py+++ b/tests/unit/test_copilot_enforcement_pruning.py@@ -599,7 +599,7 @@ def test_record_run_blocks_result_resets_stale_verified_terminal_proposal_latch(def test_record_run_blocks_result_keeps_failure_when_watchdog_cancel_without_timeout() -> None:- """Stagnation/ceiling cancels mid-session must still set last_test_ok=False+ """Ceiling cancels mid-session must still set last_test_ok=Falseso the failed-test nudge can fire — only a coincident total timeout softensto ``None`` for the unvalidated WIP rescue path."""ctx = _fresh_ctx_for_record()diff --git a/tests/unit/test_copilot_fill_credential_field.py b/tests/unit/test_copilot_fill_credential_field.pyindex 05fad3c68..4ff79f10a 100644--- a/tests/unit/test_copilot_fill_credential_field.py+++ b/tests/unit/test_copilot_fill_credential_field.py@@ -1193,21 +1193,26 @@ class TestCredentialFillInCallSubmit:assert "matches 2 controls" in result["data"]["submit_skipped"]@pytest.mark.asyncio- async def test_a_failed_submit_click_never_costs_the_fill(self, monkeypatch: pytest.MonkeyPatch) -> None:- page = _FakePage(click_error=RuntimeError("no element matched #verifyButton for 123456"))- _wire_impl(monkeypatch, page, secret_value="123456")+ @pytest.mark.parametrize(("field", "secret"), [("totp", "123456"), ("password", "Generated-Pa55word!xyz")])+ async def test_a_failed_submit_click_never_costs_the_fill(+ self, monkeypatch: pytest.MonkeyPatch, field: str, secret: str+ ) -> None:+ page = _FakePage(click_error=RuntimeError(f"no element matched #verifyButton for {secret}"))+ _wire_impl(monkeypatch, page, secret_value=secret)ctx = _ctx()- result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp", "#verifyButton")+ result = await tools_module._fill_credential_field_impl(ctx, f"#{field}", "cred_123", field, "#verifyButton")assert result["ok"] is True- assert result["data"]["typed_length"] == 6- assert "123456" not in json.dumps(result)+ assert result["data"]["typed_length"] == len(secret)+ assert secret not in json.dumps(result)assert "[REDACTED_SECRET]" in result["data"]["submit_error"]# The click raised, so whether it reached the page is unknown. Saying only "not submitted"# would read as safe to retry, and a retry spends a second code.assert result["data"]["submitted"] is Falseassert result["data"]["submit_uncertain"] is True+ assert "may already have been submitted" in result["data"]["submit_error"]+ assert page.click_calls == [("#verifyButton",)]assert [entry["tool_name"] for entry in ctx.scout_trajectory] == ["fill_credential_field"]@pytest.mark.asynciodiff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.pyindex c281cbfff..741be3b10 100644--- a/tests/unit/test_copilot_frontier.py+++ b/tests/unit/test_copilot_frontier.py@@ -160,6 +160,12 @@ class _FakeWorkflow:return copy.deepcopy(self) if deep else _FakeWorkflow(self.workflow_definition)+async def _restore_the_offered_proposal(ctx: CopilotContext, **_: object) -> None:+ # The route only admits the Test action over a pending proposal, which restore stages.+ ctx.staged_workflow = _FakeWorkflow(_wf_def())+ ctx.staged_workflow_yaml = ctx.workflow_yaml++class _FakeStream:async def is_disconnected(self) -> bool:return False@@ -4523,6 +4529,7 @@ async def test_noncompleted_test_result_handoff_survives_provider_input_merge_anelse AsyncMock(return_value=run_result))monkeypatch.setattr(agent_module, "run_workflow_end_to_end", run)+ monkeypatch.setattr(agent_module, "restore_pending_workflow_proposal", _restore_the_offered_proposal)monkeypatch.setattr(agent_module, "_resolve_live_browser_session_id", AsyncMock(return_value=None))monkeypatch.setattr("agents.mcp.MCPServerManager", FakeMCPServerManager)monkeypatch.setattr(@@ -4827,6 +4834,7 @@ async def test_completed_test_result_handoff_uses_ordinary_acting_agent_surface(return resultmonkeypatch.setattr(agent_module, "run_workflow_end_to_end", fake_run)+ monkeypatch.setattr(agent_module, "restore_pending_workflow_proposal", _restore_the_offered_proposal)monkeypatch.setattr(agent_module, "_resolve_live_browser_session_id", AsyncMock(return_value=None))monkeypatch.setattr("agents.mcp.MCPServerManager", FakeMCPServerManager)monkeypatch.setattr(diff --git a/tests/unit/test_copilot_locator_inspection.py b/tests/unit/test_copilot_locator_inspection.pyindex eae56846e..3ccbec17f 100644--- a/tests/unit/test_copilot_locator_inspection.py+++ b/tests/unit/test_copilot_locator_inspection.py@@ -43,6 +43,8 @@ from skyvern.forge.sdk.copilot.tools.locator_inspection import ()from skyvern.forge.sdk.copilot.tools.run_execution import build_test_evidence_packetfrom skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock+from skyvern.forge.sdk.workflow.models.block import CodeBlock+from skyvern.forge.sdk.workflow.models.parameter import OutputParameterfrom skyvern.schemas.workflows import BlockTypefrom tests.unit.copilot_test_helpers import (SENSITIVE_DISCLOSURE_WITHHOLDING_ARMS,@@ -905,6 +907,7 @@ async def test_prior_run_result_uses_its_exact_failed_row_and_workflow_for_typedfailure_reason="click failed",created_at=datetime(2026, 4, 21, 12, 0),trigger_type=None,+ workflow_definition_sha256=None,)now = datetime.now(UTC)block = WorkflowRunBlock(@@ -921,15 +924,24 @@ async def test_prior_run_result_uses_its_exact_failed_row_and_workflow_for_typedmodified_at=now,)workflow = SimpleNamespace(+ workflow_id="w_prior",+ workflow_permanent_id=ctx.workflow_permanent_id,+ version=1,created_by=None,modified_at=datetime(2026, 4, 21, 12, 0),workflow_definition=SimpleNamespace(blocks=[- {- "label": "click_submit",- "block_type": "code",- "code": 'await page.locator("#submit").click()',- }+ CodeBlock(+ label="click_submit",+ code='await page.locator("#submit").click()',+ output_parameter=OutputParameter(+ output_parameter_id="op_click_submit",+ key="click_submit_output",+ workflow_id="w_prior",+ created_at=now,+ modified_at=now,+ ),+ )]),)diff --git a/tests/unit/test_copilot_narration.py b/tests/unit/test_copilot_narration.pyindex f3a3ad3c9..20e5a74e6 100644--- a/tests/unit/test_copilot_narration.py+++ b/tests/unit/test_copilot_narration.py@@ -94,13 +94,12 @@ async def test_actual_run_polling_bridge_emits_progress_while_action_is_pending(nonlocal readsreads += 1if reads == 1:- return SimpleNamespace(status="running", modified_at=stamp), stamp, stamp+ return SimpleNamespace(status="running", modified_at=stamp), stampif reads == 2:- return SimpleNamespace(status="running", modified_at=stamp), stamp, stamp + timedelta(seconds=1)+ return SimpleNamespace(status="running", modified_at=stamp), stamp + timedelta(seconds=1)await release.wait()return (SimpleNamespace(status="completed", modified_at=stamp, failure_reason=None),- stamp,stamp + timedelta(seconds=2),)diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex a1be8dc0b..0e7b0b28f 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -360,7 +360,7 @@ def test_watchdog_model_facing_text_is_allowed_as_a_run_result_reply() -> None:request_policy=None,response_type="REPLY",user_response=asyncio.run(- _watchdog_error_message("stagnation", SimpleNamespace(), "wr_123", None, 300, dispatch_to_worker=True)+ _watchdog_error_message("ceiling", SimpleNamespace(), "wr_123", None, 300, dispatch_to_worker=True)),output_kind=CopilotOutputKind.WORKFLOW_RUN_RESULT,)diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.pyindex 6f995faf7..b5ac583dd 100644--- a/tests/unit/test_copilot_output_utils.py+++ b/tests/unit/test_copilot_output_utils.py@@ -1411,25 +1411,25 @@ class TestFormatToolResultForUser:result = {"ok": False,"error": (- "The run has not made progress. Run ID: wr_stalled. Outcome is uncertain. "- "Do NOT re-invoke block-running tools without first calling get_run_results."+ "The run did not reach a terminal status within the 1190s absolute ceiling. Run ID: wr_stalled. "+ "Outcome is uncertain. Do NOT re-invoke block-running tools without first calling get_run_results."),"data": {"failure_reason": (- "The run stopped after no observable progress for 120s. Run ID: wr_stalled. Outcome is uncertain."+ "The run did not finish within the 1190s absolute ceiling. Run ID: wr_stalled. Outcome is uncertain."),"control_signal": {- "kind": "watchdog_stagnation",- "user_facing_summary": "The run stopped after no observable progress for 120s.",+ "kind": "watchdog_ceiling",+ "user_facing_summary": "The run did not finish within the 1190s absolute ceiling.",},- "user_facing_summary": "The run stopped after no observable progress for 120s.",+ "user_facing_summary": "The run did not finish within the 1190s absolute ceiling.",},}summary = self._format("run_blocks_and_collect_debug", result)detail = summarize_tool_result_detail(result, tool_name="run_blocks_and_collect_debug")- assert summary == "The run stopped after no observable progress for 120s."+ assert summary == "The run did not finish within the 1190s absolute ceiling."assert detail == summaryassert "wr_stalled" not in summaryassert "get_run_results" not in detaildiff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.pyindex e534dc869..8a93c061a 100644--- a/tests/unit/test_copilot_per_tool_budget.py+++ b/tests/unit/test_copilot_per_tool_budget.py@@ -208,11 +208,11 @@ def test_watchdog_cancel_with_stale_challenge_markup_is_not_promoted() -> None:}result = {"ok": False,- "error": "Run canceled after 90s of stagnation while a Cloudflare interstitial was displayed.",+ "error": "Run canceled at the safety ceiling while a Cloudflare interstitial was displayed.","data": {"workflow_run_id": "wr_cancelled","overall_status": "canceled",- "failure_reason": "Run canceled after stagnation.",+ "failure_reason": "Run canceled at the safety ceiling.","blocks": [],},}diff --git a/tests/unit/test_copilot_positive_evidence_carriers.py b/tests/unit/test_copilot_positive_evidence_carriers.pyindex 43260a32c..511600020 100644--- a/tests/unit/test_copilot_positive_evidence_carriers.py+++ b/tests/unit/test_copilot_positive_evidence_carriers.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import jsonimport osfrom collections.abc import Sequencefrom dataclasses import FrozenInstanceError@@ -12,6 +13,7 @@ from unittest.mock import AsyncMockimport pytestfrom skyvern.forge.sdk.artifact.models import ArtifactType+from skyvern.forge.sdk.copilot import tools as tools_modulefrom skyvern.forge.sdk.copilot.build_test_outcome import ChallengeEffects, challenge_noticesfrom skyvern.forge.sdk.copilot.challenge_evidence import stamp_challenge_frame_factfrom skyvern.forge.sdk.copilot.completion_output_grounding import (@@ -51,6 +53,9 @@ from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.block import _register_code_block_secretfrom tests.unit.copilot_test_helpers import (DISPATCHED_NAV_ONLY_HTML,+ harness_run,+ historical_code_version,+ install_historical_run,make_completion_criterion,make_copilot_ctx,)@@ -1068,8 +1073,12 @@ async def test_origin_registry_recovers_when_a_later_terminal_read_finds_the_hanbrowser_session_id=None,created_at=datetime(2026, 4, 21, 12, 0),trigger_type=None,+ workflow_definition_sha256=None,)workflow = SimpleNamespace(+ workflow_id="wf_origin",+ workflow_permanent_id=ctx.workflow_permanent_id,+ version=1,created_by=None,modified_at=datetime(2026, 4, 21, 12, 0),workflow_definition=SimpleNamespace(parameters=[], blocks=[]),@@ -2174,3 +2183,132 @@ def test_runtime_output_scalar_still_credits_normally() -> None:# untouched, so the reject does not swallow the legitimate path.verdict = grade_requested_output_criteria(_star_ctx(), [_exact_value_criterion("22600")], _star_snapshot())[0]assert verdict.state == "satisfied"+++_EXECUTED_SECRET = "Rg7vQ2mXpL9sKd4TzW8nYb3HcJ5fUa6E"+++async def _run_results_page(ctx: CopilotContext, **arguments: object) -> str:+ return await tools_module.get_run_results_tool.on_invoke_tool(+ SimpleNamespace(context=ctx, tool_name="get_run_results"), json.dumps(arguments)+ )+++async def test_a_registered_secret_in_executed_source_or_defaults_reaches_no_page_or_packet(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Dense enough that the receipt's cut lands inside an occurrence, as it would if scrubbing ran after cutting.+ secret_lines = [f'v{index} = "{_EXECUTED_SECRET}"' for index in range(400)]+ code = "\n".join([f'token = "{_EXECUTED_SECRET}" + legacy_applicant_name', *secret_lines]) + "\n"+ # The secret straddles the 120-character label cut.+ label = "greet_" + "x" * 104 + _EXECUTED_SECRET+ install_historical_run(+ monkeypatch,+ historical_code_version({label: code}, default_value=_EXECUTED_SECRET),+ failed_label=label,+ )+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args: None)+ ctx = make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml="")+ ctx.secret_scrub_values = [_EXECUTED_SECRET]++ first_page = await _run_results_page(ctx, workflow_run_id="wr-1")+ detail_page = await _run_results_page(ctx, workflow_run_id="wr-1", row_keys=[f"wrb_{label}"])+ hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")++ source = json.loads(first_page)["data"]["execution_source"]+ assert source["disposition"] == "available"+ assert source["failed_statement"].endswith("+ legacy_applicant_name")+ assert source["failed_block_code_truncated"] is True+ assert hydrated is not None and hydrated["run"]["execution_source"] == source+ fragments = {_EXECUTED_SECRET[start : start + 8] for start in range(len(_EXECUTED_SECRET) - 7)}+ for projection in (first_page, detail_page, json.dumps(hydrated)):+ assert not [fragment for fragment in fragments if fragment in projection]+++async def test_a_scrubbed_multiline_secret_withholds_the_failed_statement_instead_of_naming_a_shifted_line(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ multiline_secret = "-----BEGIN KEY-----\nRg7vQ2mXpL9sKd4T\nzW8nYb3HcJ5fUa6E\n-----END KEY-----"+ filler = "".join(f"unrelated_{index} = {index}\n" for index in range(5))+ code = f'key = """{multiline_secret}"""\ngreeting = "Hello " + legacy_applicant_name\n{filler}'+ install_historical_run(+ monkeypatch,+ historical_code_version({"greet_applicant": code}),+ failed_label="greet_applicant",+ failing_line=5,+ )+ ctx = make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml="")+ ctx.secret_scrub_values = [multiline_secret]++ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)++ source = result["data"]["execution_source"]+ assert source["disposition"] == "available"+ assert "failed_statement" not in source+ assert "legacy_applicant_name" in source["failed_block_code"]+ assert "Rg7vQ2mX" not in json.dumps(source)+++async def test_a_pathological_executed_version_keeps_identity_and_statement_within_the_receipt_budget(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ statement = "greeting = " + " + ".join(["legacy_applicant_name"] * 60)+ code = statement + "\n" + "\n".join(f"step_{index} = {index}" for index in range(2_000)) + "\n"+ install_historical_run(+ monkeypatch,+ historical_code_version(+ {"greet_applicant": code}, parameter_keys=[f"{index:03d}_{'k' * 300}" for index in range(300)]+ ),+ failed_label="greet_applicant",+ )++ result = await run_execution_module._get_run_results(+ {"workflow_run_id": "wr-1"}, make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml="")+ )++ source = result["data"]["execution_source"]+ assert len(json.dumps(source)) <= 8_000+ assert (source["workflow_run_id"], source["workflow_id"], source["workflow_version"]) == ("wr-1", "wf-1", 3)+ assert source["failed_statement"] == statement[:600]+ assert source["failed_statement_truncated"] is True+ assert source["declared_parameter_keys_omitted"] > 0+ assert all(len(key) <= 120 for key in source["declared_parameter_keys"])+++async def test_escaped_text_and_labels_sharing_a_cut_stay_within_the_receipt_and_are_counted(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # JSON escapes each supplementary-plane character to 12 characters, so code-point caps alone overflow.+ failed_label = "\U0001f600" * 200+ shared_prefix = "x" * 120+ statement = "greeting = " + "\U0001f600" * 700+ install_historical_run(+ monkeypatch,+ historical_code_version({failed_label: statement + "\n" + "step = 1\n" * 2_000}),+ failed_label=failed_label,+ )++ result = await run_execution_module._get_run_results(+ {"workflow_run_id": "wr-1"}, make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml="")+ )++ source = result["data"]["execution_source"]+ assert len(json.dumps(source)) <= 8_000+ assert len(json.dumps(source["failed_block_label"])) - 2 <= 120+ assert len(json.dumps(source["failed_statement"])) - 2 <= 600+ assert source["failed_statement_truncated"] is True++ colliding = run_execution_module._run_version_source(+ make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml=""),+ harness_run("wr-1"),+ historical_code_version({shared_prefix + "_first": "pass\n", shared_prefix + "_second": "pass\n"}),+ failed_block_label=None,+ failing_line=None,+ requested_block_labels=[shared_prefix + "_first", shared_prefix + "_second"],+ )+ assert list(colliding["block_code_sha256"]) == [shared_prefix]+ assert colliding["code_block_labels_omitted"] == 1diff --git a/tests/unit/test_copilot_watchdog.py b/tests/unit/test_copilot_watchdog.pyindex 78c04605a..acc918e92 100644--- a/tests/unit/test_copilot_watchdog.py+++ b/tests/unit/test_copilot_watchdog.py@@ -1,33 +1,10 @@-"""Tests for the SKY-9163 progress-based watchdog inside-``_run_blocks_and_collect_debug``.--The full function does too much setup (prepare_workflow, execute_workflow,-parameter-binding invariants) to unit-test end-to-end cheaply. Instead we-target the isolated watchdog surface:--- ``_progress_marker`` — marker stability and field-change sensitivity.-- ``_read_progress_sources`` — correct delegation + graceful handling of- DB failures.-- ``_watchdog_error_message`` — the regression-guard strings (no- "timed out", reconciliation-instruction, per-reason body).--Those three are where the SKY-9163 correctness properties live:--1. A stale marker must be exactly equal across two polls when nothing- changed in the DB (otherwise the watchdog would false-reset on every- poll, making stagnation detection impossible).-2. Any change in ``run.status`` / ``run.modified_at`` / ``step_ts`` /- ``block_ts`` must produce a new marker (otherwise the watchdog would- false-trip on a progressing run).-3. The error messages must not read as retry-invites — that was the- original bug. "timed out" / "likely stuck repeating failing actions"- are the exact phrases the LLM used to read as "try again".-"""+"""Tests for the run poll loop inside ``_run_blocks_and_collect_debug``: its exits, error strings and cancellation."""from __future__ import annotationsimport asyncioimport time+from collections.abc import Callablefrom datetime import UTC, datetime, timedelta, timezonefrom types import SimpleNamespacefrom typing import Any@@ -49,11 +26,8 @@ from skyvern.forge.sdk.copilot.repair_origin_run import ()from skyvern.forge.sdk.copilot.tools import (RUN_BLOCKS_SAFETY_CEILING_SECONDS,- RUN_BLOCKS_STAGNATION_WINDOW_SECONDS,WatchdogExitReason,- _any_quiet_block_requested,_fallback_page_info,- _progress_marker,_read_progress_sources,_run_blocks_and_collect_debug,_watchdog_error_message,@@ -73,9 +47,6 @@ from tests.unit.copilot_test_helpers import make_copilot_ctx, run_result_action_def _fake_run(status: str = "running", modified_at: datetime | None = None) -> Any:- """A bare-minimum stand-in for ``WorkflowRun`` — the marker only reads- ``.status`` and ``.modified_at``.- """return SimpleNamespace(status=status,modified_at=modified_at or datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc),@@ -84,62 +55,6 @@ def _fake_run(status: str = "running", modified_at: datetime | None = None) -> A)-# ----------------------------------------------------------------------------# _progress_marker: stability + per-field sensitivity.-# ------------------------------------------------------------------------------def test_progress_marker_stable_for_identical_inputs() -> None:- """If the DB reports identical values on two successive polls, the marker- must compare equal. A marker that drifts on repeated reads would make the- stagnation window unreachable."""- run = _fake_run()- step_ts = datetime(2026, 4, 21, 12, 0, 30, tzinfo=timezone.utc)- block_ts = datetime(2026, 4, 21, 12, 0, 31, tzinfo=timezone.utc)-- m1 = _progress_marker(run, step_ts, block_ts)- m2 = _progress_marker(run, step_ts, block_ts)-- assert m1 == m2---def test_progress_marker_changes_on_run_status() -> None:- run1 = _fake_run(status="running")- run2 = _fake_run(status="queued")- assert _progress_marker(run1, None, None) != _progress_marker(run2, None, None)---def test_progress_marker_changes_on_run_modified_at() -> None:- run1 = _fake_run(modified_at=datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc))- run2 = _fake_run(modified_at=datetime(2026, 4, 21, 12, 0, 1, tzinfo=timezone.utc))- assert _progress_marker(run1, None, None) != _progress_marker(run2, None, None)---def test_progress_marker_changes_on_step_ts() -> None:- run = _fake_run()- t1 = datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc)- t2 = datetime(2026, 4, 21, 12, 0, 5, tzinfo=timezone.utc)- assert _progress_marker(run, t1, None) != _progress_marker(run, t2, None)---def test_progress_marker_changes_on_block_ts() -> None:- run = _fake_run()- t1 = datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc)- t2 = datetime(2026, 4, 21, 12, 0, 5, tzinfo=timezone.utc)- assert _progress_marker(run, None, t1) != _progress_marker(run, None, t2)---def test_progress_marker_tolerates_none_run() -> None:- """A transient DB read failure can return ``run=None``. The marker must- still be hashable and comparable."""- m_none = _progress_marker(None, None, None)- assert m_none == (None, None, None, None)-- # Two consecutive failed reads produce equal markers → stagnation clock- # keeps ticking (the right behavior when we can't confirm progress).- assert _progress_marker(None, None, None) == _progress_marker(None, None, None)--# ---------------------------------------------------------------------------# _read_progress_sources: delegation + graceful DB-failure handling.# ---------------------------------------------------------------------------@@ -149,25 +64,23 @@ class _FakeTasksRepo:def __init__(self,*,- step_ts: datetime | None = None,block_ts: datetime | None = None,raise_on_call: Exception | None = None,) -> None:- self.step_ts = step_tsself.block_ts = block_tsself.raise_on_call = raise_on_callself.call_count = 0- async def get_workflow_run_progress_timestamps(+ async def get_workflow_run_block_progress_timestamp(self,*,workflow_run_id: str,organization_id: str | None = None,- ) -> tuple[datetime | None, datetime | None]:+ ) -> datetime | None:self.call_count += 1if self.raise_on_call is not None:raise self.raise_on_call- return self.step_ts, self.block_ts+ return self.block_tsclass _FakeWorkflowRunsRepo:@@ -203,19 +116,14 @@ async def test_read_progress_sources_returns_run_and_timestamps(from skyvern.forge import app as forge_apprun = _fake_run()- step_ts = datetime(2026, 4, 21, 12, 0, 10, tzinfo=timezone.utc)block_ts = datetime(2026, 4, 21, 12, 0, 11, tzinfo=timezone.utc)db = _FakeDatabase(- tasks=_FakeTasksRepo(step_ts=step_ts, block_ts=block_ts),+ tasks=_FakeTasksRepo(block_ts=block_ts),workflow_runs=_FakeWorkflowRunsRepo(run=run),)monkeypatch.setattr(forge_app, "DATABASE", db)- read_run, read_step_ts, read_block_ts = await _read_progress_sources(_FakeCtx(), "wr_1")-- assert read_run is run- assert read_step_ts == step_ts- assert read_block_ts == block_ts+ assert await _read_progress_sources(_FakeCtx(), "wr_1") == (run, block_ts)@pytest.mark.asyncio@@ -227,25 +135,20 @@ async def test_read_progress_sources_swallows_workflow_run_errors(from skyvern.forge import app as forge_appdb = _FakeDatabase(- tasks=_FakeTasksRepo(step_ts=None, block_ts=None),+ tasks=_FakeTasksRepo(block_ts=None),workflow_runs=_FakeWorkflowRunsRepo(raise_on_call=RuntimeError("DB flake")),)monkeypatch.setattr(forge_app, "DATABASE", db)- read_run, read_step_ts, read_block_ts = await _read_progress_sources(_FakeCtx(), "wr_1")-- assert read_run is None- assert read_step_ts is None- assert read_block_ts is None+ assert await _read_progress_sources(_FakeCtx(), "wr_1") == (None, None)@pytest.mark.asyncioasync def test_read_progress_sources_swallows_progress_timestamps_errors(monkeypatch: pytest.MonkeyPatch,) -> None:- """A DB read failure on the aggregate timestamps must also not crash — the- caller still gets the run (if readable) and ``None`` for the timestamps.- """+ """A DB read failure on the block timestamp must also not crash — the caller still gets the run+ (if readable) and ``None`` for the timestamp."""from skyvern.forge import app as forge_apprun = _fake_run()@@ -255,11 +158,7 @@ async def test_read_progress_sources_swallows_progress_timestamps_errors()monkeypatch.setattr(forge_app, "DATABASE", db)- read_run, read_step_ts, read_block_ts = await _read_progress_sources(_FakeCtx(), "wr_1")-- assert read_run is run- assert read_step_ts is None- assert read_block_ts is None+ assert await _read_progress_sources(_FakeCtx(), "wr_1") == (run, None)# ---------------------------------------------------------------------------@@ -336,28 +235,9 @@ async def test_fallback_page_info_bounds_a_title_that_never_resolves_and_keeps_tassert page_title == ""-async def test_stagnation_error_message_does_not_invite_retry() -> None:- """The exact SKY-9163 bug: the old copy said "likely stuck repeating- failing actions" which the LLM read as "try again". The stagnation- message must explicitly discourage retry."""- msg = await _watchdog_error_message(- "stagnation", _ErrorCtx(), "wr_test", _fake_run(), RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10- )-- assert "timed out" not in msg.lower()- assert "likely stuck repeating" not in msg.lower()- assert str(RUN_BLOCKS_STAGNATION_WINDOW_SECONDS) in msg- assert "Run ID: wr_test" in msg- assert "get_run_results" in msg- assert "Do NOT re-invoke block-running tools" in msg--@pytest.mark.asyncioasync def test_ceiling_error_message_advises_splitting() -> None:- """The ceiling path is rare (a runaway run that keeps making progress- past 20 min). Its error must tell the LLM to split the workflow, not- retry — a longer run won't fit either."""+ """The error must tell the LLM to split the workflow, not retry — a longer run won't fit either."""quiet_budget = RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10msg = await _watchdog_error_message("ceiling", _ErrorCtx(), "wr_test", _fake_run(), quiet_budget)@@ -366,6 +246,7 @@ async def test_ceiling_error_message_advises_splitting() -> None:assert "split" in msg.lower()assert "Run ID: wr_test" in msgassert "get_run_results" in msg+ assert "Do NOT re-invoke block-running tools" in msg@pytest.mark.asyncio@@ -417,7 +298,7 @@ async def test_paused_error_message_reports_a_wait_not_an_uncertain_outcome() ->async def test_non_paused_error_messages_keep_the_run_id_for_the_model() -> None:"""The other arms never direct a relay — they tell the model to look the run up — so strippingthe id there would take away the only handle it has."""- exit_reasons: tuple[WatchdogExitReason, ...] = ("stagnation", "ceiling", "task_exit_unfinalized")+ exit_reasons: tuple[WatchdogExitReason, ...] = ("ceiling", "task_exit_unfinalized")for exit_reason in exit_reasons:msg = await _watchdog_error_message(exit_reason, _ErrorCtx(), "wr_test", _fake_run(), 240)@@ -433,15 +314,10 @@ async def test_non_paused_error_messages_keep_the_run_id_for_the_model() -> None_fake_run(status="paused"),"The run is paused, waiting for a person to approve or reject it.",),- (- "stagnation",- _fake_run(),- f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s.",- ),("ceiling",_fake_run(),- f"The run exceeded the {RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10}s absolute ceiling while still showing progress.",+ f"The run did not finish within the {RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10}s absolute ceiling.",),("task_exit_unfinalized",@@ -465,107 +341,6 @@ def test_watchdog_user_relayed_text_is_id_free_and_clears_the_output_guard(assert_clean_user_facing_text(reason)-# ----------------------------------------------------------------------------# _any_quiet_block_requested: stagnation bypass for block types that-# legitimately do long-silent work. Without this bypass, a WAIT block with-# wait_sec >= 90, a slow TEXT_PROMPT LLM call, or a HumanInteractionBlock-# pausing for user input would be falsely reported as stagnation and the-# tool would cancel a healthy run.-# ------------------------------------------------------------------------------def _workflow_with_block_types(*type_value_label_pairs: tuple[str, str]) -> Any:- """Build a minimal `last_workflow`-shaped object that- ``_any_quiet_block_requested`` can walk. Each pair is- ``(block_type_value, label)`` — e.g. ``("wait", "pause1")``.- """- blocks = [- SimpleNamespace(label=label, block_type=SimpleNamespace(value=block_type_value))- for block_type_value, label in type_value_label_pairs- ]- definition = SimpleNamespace(blocks=blocks)- return SimpleNamespace(workflow_definition=definition)---def test_any_quiet_block_requested_wait() -> None:- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("wait", "pause1")))- assert _any_quiet_block_requested(ctx, ["pause1"]) is True---def test_any_quiet_block_requested_text_prompt() -> None:- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("text_prompt", "prompt1")))- assert _any_quiet_block_requested(ctx, ["prompt1"]) is True---def test_any_quiet_block_requested_human_interaction() -> None:- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("human_interaction", "wait_for_user")))- assert _any_quiet_block_requested(ctx, ["wait_for_user"]) is True---def test_any_quiet_block_requested_file_download() -> None:- """File-download blocks can legitimately wait longer than the stagnation- window while the browser is waiting for the download to finish."""- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("file_download", "download_file")))- assert _any_quiet_block_requested(ctx, ["download_file"]) is True---def test_any_quiet_block_requested_code() -> None:- """A code block writes its row on entry and exit and nothing between, so a login or a long- wait inside one reads as no progress at all and the watchdog cancels a healthy run."""- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("code", "login_and_extract")))- assert _any_quiet_block_requested(ctx, ["login_and_extract"]) is True---def test_any_quiet_block_requested_mixed_requested_labels_match_quiet_one() -> None:- """When multiple blocks are requested, having any one quiet type is- enough to disable stagnation for the whole invocation."""- ctx = SimpleNamespace(- last_workflow=_workflow_with_block_types(- ("navigation", "nav1"),- ("wait", "pause1"),- ("extraction", "extract1"),- )- )- assert _any_quiet_block_requested(ctx, ["nav1", "pause1", "extract1"]) is True---def test_any_quiet_block_requested_only_task_blocks_returns_false() -> None:- """The normal case: task-heavy workflows produce regular step writes.- Stagnation is safe to enable."""- ctx = SimpleNamespace(- last_workflow=_workflow_with_block_types(- ("navigation", "nav1"),- ("extraction", "extract1"),- )- )- assert _any_quiet_block_requested(ctx, ["nav1", "extract1"]) is False---def test_any_quiet_block_requested_label_not_in_requested_ignored() -> None:- """A WAIT block defined in the workflow but not requested in this- invocation must not disable stagnation."""- ctx = SimpleNamespace(- last_workflow=_workflow_with_block_types(- ("wait", "not_requested_pause"),- ("navigation", "requested_nav"),- )- )- assert _any_quiet_block_requested(ctx, ["requested_nav"]) is False---def test_any_quiet_block_requested_no_workflow_returns_false() -> None:- """Defensive: no workflow loaded → no bypass. The loop will use its- default stagnation behavior (safe for the common case)."""- ctx = SimpleNamespace(last_workflow=None)- assert _any_quiet_block_requested(ctx, ["anything"]) is False---def test_any_quiet_block_requested_empty_labels_returns_false() -> None:- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("wait", "pause1")))- assert _any_quiet_block_requested(ctx, None) is False- assert _any_quiet_block_requested(ctx, []) is False--_HUMAN_INTERACTION_WORKFLOW_YAML = """title: human approval exampleworkflow_definition:@@ -594,6 +369,17 @@ workflow_definition:data_extraction_goal: Extract the page heading."""+_NAVIGATION_WORKFLOW_YAML = """+title: navigation example+workflow_definition:+ parameters: []+ blocks:+ - block_type: navigation+ label: open_page+ url: https://example.com+ navigation_goal: Open the page.+"""+_CODE_WORKFLOW_YAML = """title: code exampleworkflow_definition:@@ -610,6 +396,18 @@ def _adopted_detached_tasks(before: set[Any]) -> list[Any]:return [task for task in run_execution._DETACHED_CLEANUP_TASKS if task not in before]+def _install_advancing_clock(monkeypatch: pytest.MonkeyPatch, step_seconds: float = 600.0) -> Callable[[], float]:+ elapsed = 0.0++ def _monotonic() -> float:+ nonlocal elapsed+ elapsed += step_seconds+ return elapsed++ monkeypatch.setattr(run_execution, "time", SimpleNamespace(monotonic=_monotonic))+ return lambda: elapsed++@pytest.mark.asyncioasync def test_paused_run_is_reported_as_a_pause_and_left_running(monkeypatch: pytest.MonkeyPatch) -> None:"""A run paused at a human_interaction block with nobody responding: the watchdog must leave@@ -621,6 +419,7 @@ async def test_paused_run_is_reported_as_a_pause_and_left_running(monkeypatch: pworkflow_yaml=_HUMAN_INTERACTION_WORKFLOW_YAML,polled_status="paused",)+ _install_advancing_clock(monkeypatch)ctx = make_copilot_ctx(browser_session_id="pbs_chat")ctx.staged_workflow = harness["workflow"]ctx.frontier_resume_session_id = "pbs_run"@@ -730,7 +529,7 @@ async def test_a_watchdog_terminated_run_still_carries_its_per_block_page_facts(run_result_action_row("tsk_search", ActionType.CLICK, ActionStatus.completed),],)- monkeypatch.setattr(run_execution, "RUN_BLOCKS_STAGNATION_WINDOW_SECONDS", 0)+ _install_advancing_clock(monkeypatch)ctx = make_copilot_ctx(browser_session_id="pbs_chat")ctx.staged_workflow = harness["workflow"]ctx.frontier_resume_session_id = "pbs_run"@@ -740,7 +539,7 @@ async def test_a_watchdog_terminated_run_still_carries_its_per_block_page_facts()data = result["data"]- assert data["control_signal"]["kind"] == "watchdog_stagnation"+ assert data["control_signal"]["kind"] == "watchdog_ceiling"assert data["observed_block_end_urls"] == {"run_search": "https://fixture.test/results/widget/page-1","select_first_result": "https://fixture.test/results/widget",@@ -752,14 +551,14 @@ async def test_a_watchdog_terminated_run_still_carries_its_per_block_page_facts(@pytest.mark.asyncioasync def test_non_paused_watchdog_exit_still_cancels_and_clears(monkeypatch: pytest.MonkeyPatch) -> None:- """The pause carve-out is scoped to the pause: a stagnating run still gets cancelled and still- releases the run-session association."""+ """The pause carve-out is scoped to the pause: a run that reaches the ceiling still gets cancelled+ and still releases the run-session association."""harness = await _install_run_harness(monkeypatch,workflow_yaml=_EXTRACTION_WORKFLOW_YAML,polled_status="running",)- monkeypatch.setattr(run_execution, "RUN_BLOCKS_STAGNATION_WINDOW_SECONDS", 0)+ _install_advancing_clock(monkeypatch)ctx = make_copilot_ctx(browser_session_id="pbs_chat")ctx.staged_workflow = harness["workflow"]ctx.frontier_resume_session_id = "pbs_run"@@ -767,7 +566,7 @@ async def test_non_paused_watchdog_exit_still_cancels_and_clears(monkeypatch: pyresult = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)- assert result["data"]["control_signal"]["kind"] == "watchdog_stagnation"+ assert result["data"]["control_signal"]["kind"] == "watchdog_ceiling"harness["cancel_run_task"].assert_awaited_once()harness["clear"].assert_awaited_once()assert _adopted_detached_tasks(before) == []@@ -813,8 +612,7 @@ async def test_non_success_watchdog_result_types_selected_failed_block_locators()observe = AsyncMock(return_value=[{"authored_selector": "#submit", "unobserved_reason": "run_page_unavailable"}])monkeypatch.setattr(run_execution, "_observe_authored_locators", observe)- monkeypatch.setattr(run_execution, "RUN_BLOCKS_STAGNATION_WINDOW_SECONDS", 0)- monkeypatch.setattr(run_execution, "_any_quiet_block_requested", lambda *_args, **_kwargs: False)+ _install_advancing_clock(monkeypatch)ctx = make_copilot_ctx(browser_session_id="pbs_chat")ctx.staged_workflow = harness["workflow"]ctx.frontier_resume_session_id = "pbs_run"@@ -834,10 +632,8 @@ async def test_non_success_watchdog_result_types_selected_failed_block_locators(@pytest.mark.asyncio[email protected]("_repeat", range(3))async def test_progressing_worker_run_crosses_legacy_boundary_and_returns_terminal_result(monkeypatch: pytest.MonkeyPatch,- _repeat: int,) -> None:harness = await _install_run_harness(monkeypatch,@@ -872,11 +668,11 @@ async def test_progressing_worker_run_crosses_legacy_boundary_and_returns_termin))- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime, datetime]:+ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime]:nonlocal elapsedelapsed, status = next(progress)marker = datetime(2026, 4, 21, 12, 0, 0, tzinfo=UTC) + timedelta(seconds=elapsed)- return _fake_run(status=status, modified_at=marker), marker, marker+ return _fake_run(status=status, modified_at=marker), markermonkeypatch.setattr(run_execution, "_read_progress_sources", _read_progress)monkeypatch.setattr(run_execution, "time", SimpleNamespace(monotonic=lambda: elapsed))@@ -908,59 +704,20 @@ async def test_progressing_worker_run_crosses_legacy_boundary_and_returns_termin@pytest.mark.asyncio[email protected]("_repeat", range(3))[email protected]("layout", ["direct", "loop", "nested", "finally"])-async def test_web_search_finishes_after_silent_period_without_watchdog_cancellation(+ ("workflow_yaml", "label", "block_type"),+ [+ (_NAVIGATION_WORKFLOW_YAML, "open_page", BlockType.NAVIGATION),+ (_EXTRACTION_WORKFLOW_YAML, "extract_heading", BlockType.EXTRACTION),+ ],+ ids=["navigation", "extraction"],+)+async def test_silent_worker_run_finishes_pending_model_call_without_cancel(monkeypatch: pytest.MonkeyPatch,- _repeat: int,- layout: str,+ workflow_yaml: str,+ label: str,+ block_type: BlockType,) -> None:- workflow_yaml = """-title: web search example-workflow_definition:- parameters: []- blocks:- - block_type: web_search- label: search- query: site:example.com docs-"""- selected_label = "search"- if layout in {"loop", "nested"}:- workflow_yaml = """-title: web search loop-workflow_definition:- parameters:- - parameter_type: workflow- key: items- workflow_parameter_type: json- default_value: [1]- blocks:- - block_type: for_loop- label: loop- loop_over_parameter_key: items- loop_blocks:- - block_type: web_search- label: search- query: site:example.com docs-"""- selected_label = "search" if layout == "nested" else "loop"- elif layout == "finally":- workflow_yaml = """-title: web search cleanup-workflow_definition:- parameters: []- finally_block_label: search- blocks:- - block_type: navigation- label: navigate- url: https://example.com- navigation_goal: Open the page.- - block_type: web_search- label: search- query: site:example.com docs-"""- selected_label = "navigate"- output = {"query": "site:example.com docs", "results": [], "total_count": 0, "prompt_output": None}harness = await _install_run_harness(monkeypatch,workflow_yaml=workflow_yaml,@@ -968,26 +725,33 @@ workflow_definition:dispatch_to_worker=True,terminal_blocks=[WorkflowRunBlock(- label="search",- block_type=BlockType.WEB_SEARCH,+ label=label,+ block_type=block_type,status="completed",- output=output,workflow_run_block_id="wrb_terminal",workflow_run_id="wr_paused",organization_id="org-1",created_at=datetime(2026, 4, 21, 12, 0, tzinfo=UTC),- modified_at=datetime(2026, 4, 21, 12, 2, 30, tzinfo=UTC),+ modified_at=datetime(2026, 4, 21, 12, 3, 3, tzinfo=UTC),)],)elapsed = 0.0- progress = iter(((0.0, "running"), (120.0, "running"), (150.0, "completed")))+ progress = iter(+ (+ (0.0, "running"),+ (89.0, "running"),+ (91.0, "running"),+ (182.0, "running"),+ (183.0, "completed"),+ )+ )marker = datetime(2026, 4, 21, 12, 0, tzinfo=UTC)- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime, datetime]:+ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime]:nonlocal elapsedelapsed, status = next(progress)- return _fake_run(status=status, modified_at=marker), marker, marker+ return _fake_run(status=status, modified_at=marker), markermonkeypatch.setattr(run_execution, "_read_progress_sources", _read_progress)monkeypatch.setattr(run_execution, "time", SimpleNamespace(monotonic=lambda: elapsed))@@ -995,15 +759,81 @@ workflow_definition:ctx.staged_workflow = harness["workflow"]ctx.frontier_resume_session_id = "pbs_run"- result = await _run_blocks_and_collect_debug({"block_labels": [selected_label], "parameters": {}}, ctx)+ result = await _run_blocks_and_collect_debug({"block_labels": [label], "parameters": {}}, ctx)- assert elapsed == 150.0+ harness["cooperative_cancel"].assert_not_awaited()+ assert elapsed == 183.0assert result["ok"] is True, result+ assert result["data"]["workflow_run_id"] == "wr_paused"assert result["data"]["overall_status"] == "completed"- assert result["data"]["blocks"][0]["output"] == outputharness["worker_execute"].assert_awaited_once()+++async def test_never_terminal_worker_run_is_cancelled_at_the_safety_ceiling(monkeypatch: pytest.MonkeyPatch) -> None:+ harness = await _install_run_harness(+ monkeypatch,+ workflow_yaml=_EXTRACTION_WORKFLOW_YAML,+ polled_status="running",+ dispatch_to_worker=True,+ )+ _install_advancing_clock(monkeypatch)+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")+ ctx.staged_workflow = harness["workflow"]+ ctx.frontier_resume_session_id = "pbs_run"++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)++ assert result["ok"] is False, result+ assert result["data"]["control_signal"]["kind"] == "watchdog_ceiling"+ assert f"{RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10}s" in result["data"]["user_facing_summary"]+ harness["cooperative_cancel"].assert_awaited_once_with("wr_paused")+++async def test_terminal_row_written_before_the_ceiling_cancel_wins(monkeypatch: pytest.MonkeyPatch) -> None:+ harness = await _install_run_harness(+ monkeypatch,+ workflow_yaml=_EXTRACTION_WORKFLOW_YAML,+ polled_status="running",+ dispatch_to_worker=True,+ )+ forge_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=_fake_run(status="completed"))+ _install_advancing_clock(monkeypatch)+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")+ ctx.staged_workflow = harness["workflow"]+ ctx.frontier_resume_session_id = "pbs_run"++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)+harness["cooperative_cancel"].assert_not_awaited()- harness["cancel_run_task"].assert_not_awaited()+ assert result["ok"] is True, result+ assert result["data"]["overall_status"] == "completed"++[email protected](("status", "ok"), [("completed", True), ("failed", False)])+async def test_immediate_terminal_worker_result_is_returned_as_recorded(+ monkeypatch: pytest.MonkeyPatch, status: str, ok: bool+) -> None:+ harness = await _install_run_harness(+ monkeypatch,+ workflow_yaml=_EXTRACTION_WORKFLOW_YAML,+ polled_status=status,+ dispatch_to_worker=True,+ )+ clock = _install_advancing_clock(monkeypatch, step_seconds=1.0)+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")+ ctx.staged_workflow = harness["workflow"]+ ctx.frontier_resume_session_id = "pbs_run"++ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)++ assert clock() < RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10+ harness["cooperative_cancel"].assert_not_awaited()+ assert result["ok"] is ok, result+ assert result["data"]["overall_status"] == status+ assert "control_signal" not in result["data"]@pytest.mark.asyncio@@ -1019,14 +849,14 @@ async def test_externally_cancelled_worker_run_still_cooperatively_cancels(polling = asyncio.Event()reads = 0- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime, datetime]:+ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime]:nonlocal readsreads += 1marker = datetime(2026, 4, 21, 12, 0, reads, tzinfo=UTC)if reads > 1:polling.set()await asyncio.Event().wait()- return _fake_run(status="running", modified_at=marker), marker, marker+ return _fake_run(status="running", modified_at=marker), markermonkeypatch.setattr(run_execution, "_read_progress_sources", _read_progress)diff --git a/tests/unit/test_debug_sessions_lookup_index_migration.py b/tests/unit/test_debug_sessions_lookup_index_migration.pynew file mode 100644index 000000000..5278264d6--- /dev/null+++ b/tests/unit/test_debug_sessions_lookup_index_migration.py@@ -0,0 +1,195 @@+import functools+import importlib.util+import re+import uuid+from collections.abc import Iterator+from datetime import datetime, timedelta+from pathlib import Path+from types import ModuleType+from typing import Any, NamedTuple++import pytest+from sqlalchemy import Engine, create_engine, event, insert, text+from sqlalchemy.engine import make_url+from sqlalchemy.exc import DBAPIError+from sqlalchemy.ext.asyncio import create_async_engine+from sqlalchemy.schema import CreateTable++from alembic.operations import Operations+from alembic.runtime.migration import MigrationContext+from skyvern.config import settings+from skyvern.forge.sdk.db.agent_db import AgentDB+from skyvern.forge.sdk.db.models import DebugSessionModel++INDEX_NAME = "ix_debug_sessions_org_wpid_user_created_at"+++def _migration() -> ModuleType:+ # Matched by suffix: the open-source mirror regenerates this migration under its own date and revision id, and+ # names it after the slug of the docstring's first line, so that line must stay "index debug_sessions lookup".+ versions = Path(__file__).resolve().parents[2] / "alembic/versions"+ matches = sorted(versions.glob("*_index_debug_sessions_lookup.py"))+ if not matches:+ raise AssertionError(f"no index_debug_sessions_lookup migration found in {versions}")+ spec = importlib.util.spec_from_file_location("index_debug_sessions_lookup_migration", matches[-1])+ assert spec is not None and spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++class _Schema(NamedTuple):+ engine: Engine+ connect_args: dict[str, str]+++def scratch_schema() -> Iterator[_Schema]:+ url = make_url(str(settings.DATABASE_STRING))+ if url.get_backend_name() != "postgresql":+ pytest.skip("requires PostgreSQL")+ schema = f"debug_sessions_index_{uuid.uuid4().hex}"+ connect_args = {"options": f"-csearch_path={schema}"}+ engine = create_engine(url.set(drivername="postgresql+psycopg"), connect_args=connect_args)+ with engine.begin() as connection:+ connection.exec_driver_sql(f'CREATE SCHEMA "{schema}"')+ connection.execute(CreateTable(DebugSessionModel.__table__))+ try:+ yield _Schema(engine, connect_args)+ finally:+ with engine.begin() as connection:+ connection.exec_driver_sql(f'DROP SCHEMA "{schema}" CASCADE')+ engine.dispose()+++def _upgrade(engine: Engine) -> None:+ with engine.connect() as connection, Operations.context(MigrationContext.configure(connection)):+ _migration().upgrade()+++def _index_state(engine: Engine) -> tuple[bool, str] | None:+ with engine.connect() as connection:+ row = connection.execute(+ text(+ "SELECT indisvalid, pg_get_indexdef(indexrelid) FROM pg_catalog.pg_index "+ "WHERE indexrelid = to_regclass(:name)"+ ),+ {"name": INDEX_NAME},+ ).one_or_none()+ return None if row is None else (row[0], row[1])+++def _session_row(organization_id: str, name: str, **overrides: Any) -> dict[str, Any]:+ return {+ "debug_session_id": f"ds_{name}",+ "organization_id": organization_id,+ "browser_session_id": f"pbs_{name}",+ "workflow_permanent_id": "wpid_target",+ "user_id": "user_target",+ "status": "created",+ "deleted_at": None,+ **overrides,+ }+++def _generic_plan(engine: Engine, statement: str, parameters: dict[str, Any]) -> str:+ names: list[str] = []++ def number(match: re.Match[str]) -> str:+ if match.group(1) not in names:+ names.append(match.group(1))+ return f"${names.index(match.group(1)) + 1}"++ prepared = re.sub(r"%\((\w+)\)s", number, statement)+ arguments = ", ".join(+ f"'{value}'" if isinstance(value, str) else str(value) for value in (parameters[name] for name in names)+ )+ with engine.connect() as connection:+ connection.exec_driver_sql("SET LOCAL plan_cache_mode = force_generic_plan")+ # A seq scan always wins on a seven-row table, so disable it to ask only whether the index can serve the query.+ connection.exec_driver_sql("SET LOCAL enable_seqscan = off")+ connection.exec_driver_sql(f"PREPARE debug_session_lookup AS {prepared}")+ try:+ plan = connection.exec_driver_sql(f"EXPLAIN EXECUTE debug_session_lookup({arguments})").scalars().all()+ finally:+ connection.exec_driver_sql("DEALLOCATE debug_session_lookup")+ connection.rollback()+ return "\n".join(plan)+++async def test_lookup_returns_the_newest_open_session_through_the_index_under_a_generic_plan(+ scratch_schema: _Schema,+) -> None:+ schema_engine = scratch_schema.engine+ _upgrade(schema_engine)+ organization_id = "o_debug_lookup"+ started = datetime(2026, 9, 1)+ with schema_engine.begin() as connection:+ connection.execute(+ insert(DebugSessionModel),+ [+ _session_row(organization_id, "older_open", created_at=started),+ _session_row(organization_id, "newest_open", created_at=started + timedelta(minutes=1)),+ _session_row(+ organization_id, "completed", created_at=started + timedelta(minutes=2), status="completed"+ ),+ _session_row(organization_id, "deleted", created_at=started + timedelta(minutes=3), deleted_at=started),+ _session_row(organization_id, "other_user", created_at=started + timedelta(minutes=4), user_id="u2"),+ _session_row(+ organization_id,+ "other_workflow",+ created_at=started + timedelta(minutes=5),+ workflow_permanent_id="w2",+ ),+ _session_row("o_other", "other_org", created_at=started + timedelta(minutes=6)),+ ],+ )++ async_engine = create_async_engine(schema_engine.url, connect_args=scratch_schema.connect_args)+ statements: list[tuple[str, dict[str, Any]]] = []++ def capture(_conn: Any, _cursor: Any, statement: str, parameters: Any, *_args: Any) -> None:+ if "FROM debug_sessions" in statement:+ statements.append((statement, parameters))++ event.listen(async_engine.sync_engine, "before_cursor_execute", capture)+ try:+ found = await AgentDB(str(async_engine.url), db_engine=async_engine).debug.get_debug_session(+ organization_id=organization_id,+ user_id="user_target",+ workflow_permanent_id="wpid_target",+ )+ finally:+ event.remove(async_engine.sync_engine, "before_cursor_execute", capture)+ await async_engine.dispose()++ assert found is not None and found.debug_session_id == "ds_newest_open"+ [(statement, parameters)] = statements+ # psycopg prepares a statement after five runs on a connection, and Postgres may then plan it generically,+ # with status as an unknown parameter.+ plan = _generic_plan(schema_engine, statement, parameters)+ assert f"Index Scan Backward using {INDEX_NAME}" in plan, plan+ assert "Sort" not in plan, plan+++def test_upgrade_rebuilds_an_invalid_index_left_by_a_failed_concurrent_build(scratch_schema: _Schema) -> None:+ schema_engine = scratch_schema.engine+ with schema_engine.begin() as connection:+ connection.execute(insert(DebugSessionModel), [_session_row("o_debug_lookup", "only")])+ with schema_engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:+ with pytest.raises(DBAPIError, match="division by zero"):+ connection.exec_driver_sql(+ f"CREATE INDEX CONCURRENTLY {INDEX_NAME} ON debug_sessions ((length(user_id) / 0))"+ )+ leftover = _index_state(schema_engine)+ assert leftover is not None and leftover[0] is False++ _upgrade(schema_engine)++ state = _index_state(schema_engine)+ assert state is not None+ valid, definition = state+ assert valid is True+ assert "(organization_id, workflow_permanent_id, user_id, created_at)" in definitiondiff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.pyindex 6b448dc52..15d671aa0 100644--- a/tests/unit/test_request_logging.py+++ b/tests/unit/test_request_logging.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport ioimport json-import loggingimport typingfrom datetime import UTC, datetimefrom types import SimpleNamespace@@ -10,7 +9,6 @@ from unittest.mock import AsyncMock, MagicMockimport psycopg.errorsimport pytest-import structlogfrom fastapi import FastAPI, HTTPException, Responsefrom fastapi.responses import JSONResponse, StreamingResponsefrom fastapi.testclient import TestClient@@ -18,7 +16,7 @@ from sqlalchemy.exc import OperationalErrorfrom starlette.requests import ClientDisconnect, Requestfrom starlette.types import ASGIApp, Receive, Scope, Send-from skyvern.forge import api_app, forge_app_initializer, request_logging+from skyvern.forge import api_app, request_loggingfrom skyvern.forge.log_redaction import (REDACTED,SENSITIVE_FIELDS,@@ -39,7 +37,6 @@ from skyvern.forge.request_logging import (log_raw_request_middleware,set_request_organization,)-from skyvern.forge.sdk.forge_log import setup_loggerfrom skyvern.forge.sdk.schemas.organizations import Organizationfrom skyvern.forge.sdk.services.org_auth_service import apply_request_org_context@@ -129,6 +126,11 @@ class TestIsSensitiveKey:"client_secret","access_token","refresh_token",+ # Payment-provider payer details and webhook signatures+ "billing_details",+ "payment_method_details",+ "receipt_email",+ "Stripe-Signature",],)def test_sensitive_keys_are_redacted(self, key: str) -> None:@@ -1163,27 +1165,6 @@ _ROUTE_CALL_DEPTH = 25_TEST_ORGANIZATION_ID = "o_385835488455492960"-def rendered_log_stream(monkeypatch: pytest.MonkeyPatch) -> typing.Iterator[io.StringIO]:- monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS", True)- monkeypatch.setattr(request_logging.settings, "JSON_LOGGING", True)- root_logger = logging.getLogger()- saved_handlers = root_logger.handlers[:]- saved_structlog_config = structlog.get_config()- setup_logger()- # create_api_app configures logging once per process; keep it from replacing this handler.- monkeypatch.setattr(forge_app_initializer, "_SERVER_LOGGING_CONFIGURED", True)- stream = io.StringIO()- handler = root_logger.handlers[0]- assert isinstance(handler, logging.StreamHandler)- handler.setStream(stream)- try:- yield stream- finally:- root_logger.handlers[:] = saved_handlers- structlog.configure(**saved_structlog_config)--def _raise_from_depth(depth: int) -> None:if depth == 0:raise RuntimeError("connection pool exhausted")diff --git a/tests/unit/test_saved_session_proxy_pin.py b/tests/unit/test_saved_session_proxy_pin.pyindex 5d3b1cbbe..370dd5e16 100644--- a/tests/unit/test_saved_session_proxy_pin.py+++ b/tests/unit/test_saved_session_proxy_pin.py@@ -11,7 +11,12 @@ from skyvern.forge.sdk.schemas.browser_profiles import BrowserProfilefrom skyvern.forge.sdk.schemas.persistent_browser_sessions import FORCED_WORKFLOW_SESSION_RUNNABLE_TYPEfrom skyvern.forge.sdk.workflow.browser_profile_key import build_browser_profile_key_digestfrom skyvern.forge.sdk.workflow.models.parameter import CredentialParameter, WorkflowParameter, WorkflowParameterType-from skyvern.forge.sdk.workflow.models.workflow import WorkflowRequestBody, WorkflowRun+from skyvern.forge.sdk.workflow.models.workflow import (+ WorkflowDefinition,+ WorkflowRequestBody,+ WorkflowRun,+ workflow_definition_sha256,+)from skyvern.forge.sdk.workflow.service import WorkflowServicefrom skyvern.schemas.browser_session_kind import BrowserSessionKindfrom skyvern.schemas.proxy_pinning import derive_proxy_session_id, is_proxy_session_id@@ -46,7 +51,7 @@ def _workflow(code_version=None,adaptive_caching=False,sequential_key=None,- workflow_definition=SimpleNamespace(parameters=parameters or [], blocks=[], retry_policy=None),+ workflow_definition=WorkflowDefinition(parameters=parameters or [], blocks=[]),)@@ -792,8 +797,11 @@ async def test_create_workflow_run_non_force_path_single_create_no_update(monkey)monkeypatch.setattr(app.DATABASE.workflow_runs, "create_workflow_run", create_workflow_run)monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run", update_workflow_run)+ workflow = _workflow()+ service = WorkflowService()+ monkeypatch.setattr(service, "get_workflow", AsyncMock(return_value=workflow))- result = await WorkflowService().create_workflow_run(+ result = await service.create_workflow_run(workflow_request=request,workflow_permanent_id="wpid_test",workflow_id="wf_test",@@ -833,6 +841,7 @@ async def test_create_workflow_run_non_force_path_single_create_no_update(monkeyignore_inherited_workflow_system_prompt=False,copilot_session_id=None,created_by=None,+ workflow_definition_sha256=workflow_definition_sha256(workflow.workflow_definition),)update_workflow_run.assert_not_awaited()@@ -1200,3 +1209,13 @@ async def test_create_persistent_browser_session_does_not_inherit_profile_pin_byassert session.proxy_location is Noneassert session.proxy_session_id is Noneassert session.browser_profile_id == "bp_managed"+++async def test_a_created_run_records_the_digest_of_the_definition_it_runs(monkeypatch: pytest.MonkeyPatch) -> None:+ workflow = _workflow(parameters=[_workflow_parameter("credential_id", default_value="cred_default")])++ created = await _create_forced_workflow_run(monkeypatch, workflow=workflow)++ recorded = created.create_workflow_run.await_args.kwargs["workflow_definition_sha256"]+ assert recorded == workflow_definition_sha256(workflow.workflow_definition)diff --git a/tests/unit/test_task_v2_completion_gate.py b/tests/unit/test_task_v2_completion_gate.pyindex d2f5af296..77b56fc4e 100644--- a/tests/unit/test_task_v2_completion_gate.py+++ b/tests/unit/test_task_v2_completion_gate.py@@ -15,7 +15,7 @@ import pytestfrom skyvern.config import Settingsfrom skyvern.forge.sdk.prompting import PromptEnginefrom skyvern.forge.sdk.schemas.task_v2 import TaskV2Status-from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus+from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition, WorkflowRunStatusfrom skyvern.forge.sdk.workflow.service import WorkflowServicefrom skyvern.services import planner_levers, task_v2_servicefrom skyvern.services.task_v2_service import _should_run_post_block_completion_check@@ -39,7 +39,12 @@ async def _run_task_v2_wiring_case(task_v2.url, task_v2.workflow_run_id = "https://example.test", "wr_test"queued_run = MagicMock(workflow_run_id="wr_test", workflow_id="wf_test", status=WorkflowRunStatus.queued)running_run = MagicMock(workflow_run_id="wr_test", workflow_id="wf_test", status=WorkflowRunStatus.running)- workflow = MagicMock(workflow_id="wf_test", workflow_permanent_id="wp_test", title="Test")+ workflow = MagicMock(+ workflow_id="wf_test",+ workflow_permanent_id="wp_test",+ title="Test",+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),+ )block_result = MagicMock(failure_reason=None, output_parameter_value={}, success=completion_case)block = SimpleNamespace(execute_safe=AsyncMock(return_value=block_result))scraped_page = SimpleNamespace(screenshots=[])@@ -220,6 +225,7 @@ async def test_first_iteration_goto_url_has_no_completion_criterion(status="published",extra_http_headers=None,cdp_connect_headers=None,+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),)block_result = SimpleNamespace(status="failed",@@ -352,6 +358,7 @@ async def test_missing_task_type_failure_finalizes_with_organization_scope(title="Test workflow",description=None,status="published",+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),)page = MagicMock()scraped_page = SimpleNamespace(screenshots=[])diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.pyindex c4baf7ff5..3979bea60 100644--- a/tests/unit/test_workflow_block_engine.py+++ b/tests/unit/test_workflow_block_engine.py@@ -542,9 +542,7 @@ async def _persisted_engine_from_execute_workflow_blocks(block = block or _make_block(TaskBlock, label="e2e_block")workflow = MagicMock()- workflow.workflow_definition.blocks = [block]- workflow.workflow_definition.version = 1- workflow.workflow_definition.finally_block_label = None+ workflow.workflow_definition = WorkflowDefinition(parameters=[], blocks=[block])workflow.status = workflow_statusworkflow_run = MagicMock()diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 0c1b00c26..9afa29f62 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -101,6 +101,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatRequest,WorkflowCopilotChatSender,WorkflowCopilotClearProposedWorkflowRequest,+ WorkflowCopilotCredentialRegistration,WorkflowCopilotDisableAutoAcceptRequest,WorkflowCopilotMessageFeedbackRequest,WorkflowCopilotStreamErrorUpdate,@@ -4557,7 +4558,10 @@ async def test_request_webhook_is_validated_before_staging_once(await captured["handler"](copilot_stream)frames = [call.args[0].model_dump(mode="json") for call in copilot_stream.send.await_args_list]- if webhook_kind == "blocked":+ # The Test action runs the proposal under its own settings, so the editor's webhook is never carried.+ carried = product_action is None+ expected_webhook = private_url if carried or webhook_kind == "stored" else None+ if webhook_kind == "blocked" and carried:assert not contexts, "The agent must not receive a request with a newly carried blocked webhook"errors = [frame for frame in frames if frame["type"] == "error"]assert errors and "webhook_callback_url" in errors[0]["error"]@@ -4566,7 +4570,7 @@ async def test_request_webhook_is_validated_before_staging_once(else:assert contexts and all(update["ok"] for update in updates), (updates, frames)ctx = contexts[0]- assert ctx.staged_workflow.webhook_callback_url == private_url+ assert ctx.staged_workflow.webhook_callback_url == expected_webhookassert private_url not in ctx.workflow_yamlchat.proposed_workflow = workflow_copilot_route._build_proposed_workflow_data(ctx.staged_workflow,@@ -4584,8 +4588,8 @@ async def test_request_webhook_is_validated_before_staging_once(await workflow_copilot_chat_post(api_key_request, request, organization)await captured["handler"](copilot_stream)assert len(contexts) == 2 and all(update["ok"] for update in updates), updates- assert contexts[-1].staged_workflow.webhook_callback_url == private_url- assert validated_urls == ([] if webhook_kind == "stored" else [private_url])+ assert contexts[-1].staged_workflow.webhook_callback_url == expected_webhook+ assert validated_urls == ([private_url] if carried and webhook_kind != "stored" else [])@pytest.mark.asyncio@@ -6091,7 +6095,7 @@ async def test_reconcile_leaves_a_turn_that_just_resumed_from_a_credential_pause@pytest.mark.asyncio[email protected]("answer", ["connected", "skip", "signed_in"])[email protected]("answer", ["connected", "skip", "signed_in", "generated"])async def test_reconcile_between_a_card_answer_and_the_waiter_leaves_the_turn(sqlite_engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch, answer: str) -> None:@@ -6105,11 +6109,23 @@ async def test_reconcile_between_a_card_answer_and_the_waiter_leaves_the_turn(sign_in = credential_pause.ManualSignIn(browser_session_id="pbs_debug", login_urls=["https://portal.example.com/login"], profile_name="Sign-in")+ registration = (+ WorkflowCopilotCredentialRegistration(username="[email protected]", credential_name="Portal")+ if answer == "generated"+ else None+ )cache.store[credential_pause.credential_pause_active_key("org", chat_id, "turn-a")] = (- credential_pause._encode_active_pause(token, datetime.now(UTC) + timedelta(minutes=5), manual_sign_in=sign_in)+ credential_pause._encode_active_pause(+ token, datetime.now(UTC) + timedelta(minutes=5), manual_sign_in=sign_in, registration=registration+ ))ids = {"organization_id": "org", "workflow_copilot_chat_id": chat_id, "turn_id": "turn-a"}- if answer == "signed_in":+ if answer == "generated":+ generation = await credential_pause.claim_credential_generation(cache, resume_token=token, **ids)+ assert await credential_pause.finish_credential_generation(+ cache, claim=generation, credential_id="cred_generated", **ids+ )+ elif answer == "signed_in":claim = await credential_pause.claim_manual_sign_in(cache, resume_token=token, **ids)profile = credential_pause.SignedInProfile(browser_profile_id="bp_signed_in", profile_name="Sign-in", site="portal.example.com", cookie_count=1@@ -7020,7 +7036,8 @@ async def test_test_end_to_end_route_hands_the_proposal_bound_account_to_the_agefor key in ("extra_http_headers", "cdp_connect_headers", "totp_identifier", "totp_verification_url")}expected_settings["totp_identifier"] = "synthetic-unsaved-identifier"- assert private_settings == expected_settings+ # The Test action runs the proposal under its own settings, so the editor's unsaved ones never reach the turn.+ assert private_settings == ({} if product_action else expected_settings)expected_yaml.pop("totp_identifier")expected_yaml.pop("totp_verification_url")expected_yaml["extra_http_headers"] = {"Authorization": "***"}diff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.pyindex a5267ff10..64e6dd658 100644--- a/tests/unit/workflow/test_code_block_recorder.py+++ b/tests/unit/workflow/test_code_block_recorder.py@@ -190,6 +190,9 @@ class FakePage:def set_default_timeout(self, timeout: float) -> None:self.default_timeout = timeout+ def is_closed(self) -> bool:+ return False+async def goto(self, url, **kwargs): # noqa: ANN001, ANN003, ANN201return None