Skip to content

Automate browser based workflows with AI

Test a restored Copilot proposal on its stored bytes, with no re-save needed (SKY-17668) (#8801)

Andrew Neilson778ae699bc0e
diff --git a/alembic/versions/2026_10_04_0635-2b0a571b6e5e_index_debug_sessions_lookup.py b/alembic/versions/2026_10_04_0635-2b0a571b6e5e_index_debug_sessions_lookup.py
new file mode 100644
index 000000000..03270ed3c
--- /dev/null
+++ b/alembic/versions/2026_10_04_0635-2b0a571b6e5e_index_debug_sessions_lookup.py
@@ -0,0 +1,65 @@
+"""index debug_sessions lookup
+
+Revision ID: 2b0a571b6e5e
+Revises: 6fbee6664b86
+Create Date: 2026-10-04T06:35:37.146186+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "2b0a571b6e5e"
+down_revision: Union[str, None] = "6fbee6664b86"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+INDEX_NAME = "ix_debug_sessions_org_wpid_user_created_at"
+_TABLE = "debug_sessions"
+
+
+def _index_is_valid() -> bool | None:
+ return (
+ op.get_bind()
+ .execute(
+ sa.text("SELECT indisvalid FROM pg_catalog.pg_index WHERE indexrelid = to_regclass(:name)"),
+ {"name": INDEX_NAME},
+ )
+ .scalar_one_or_none()
+ )
+
+
+def upgrade() -> None:
+ with op.get_context().autocommit_block():
+ op.execute("SET statement_timeout = '10min';")
+ # A failed CONCURRENTLY build leaves an INVALID index of this name, which IF NOT EXISTS would keep.
+ # Assumes a single migration runner: another runner's in-flight build also reads as INVALID.
+ if _index_is_valid() is False:
+ op.execute(f"DROP INDEX CONCURRENTLY {INDEX_NAME};")
+ # Deliberately not partial on status: the lookup binds status as a parameter, so a generic plan
+ # cannot prove `status = 'created'` and would skip a partial index.
+ op.create_index(
+ INDEX_NAME,
+ _TABLE,
+ ["organization_id", "workflow_permanent_id", "user_id", "created_at"],
+ unique=False,
+ postgresql_concurrently=True,
+ if_not_exists=True,
+ )
+ op.execute("RESET statement_timeout;")
+
+
+def downgrade() -> None:
+ with op.get_context().autocommit_block():
+ op.execute("SET statement_timeout = '10min';")
+ op.drop_index(
+ INDEX_NAME,
+ table_name=_TABLE,
+ postgresql_concurrently=True,
+ if_exists=True,
+ )
+ op.execute("RESET statement_timeout;")
diff --git a/alembic/versions/2026_10_04_0635-798a79449b1e_add_workflow_definition_sha256_to_workflow_runs.py b/alembic/versions/2026_10_04_0635-798a79449b1e_add_workflow_definition_sha256_to_workflow_runs.py
new file mode 100644
index 000000000..d3a949434
--- /dev/null
+++ b/alembic/versions/2026_10_04_0635-798a79449b1e_add_workflow_definition_sha256_to_workflow_runs.py
@@ -0,0 +1,29 @@
+"""add workflow_definition_sha256 to workflow_runs
+
+Revision ID: 798a79449b1e
+Revises: 2b0a571b6e5e
+Create Date: 2026-10-04T06:35:37.147784+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "798a79449b1e"
+down_revision: Union[str, None] = "2b0a571b6e5e"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.execute("SET LOCAL lock_timeout = '5s'")
+ op.add_column("workflow_runs", sa.Column("workflow_definition_sha256", sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+ op.execute("SET LOCAL lock_timeout = '5s'")
+ op.drop_column("workflow_runs", "workflow_definition_sha256")
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx
index 7a2d22a3f..49744152f 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx
@@ -34,6 +34,7 @@ type StreamBody = {
cancel_token?: string;
supports_credential_pause?: boolean;
supports_credential_pause_recovery?: boolean;
+ supports_credential_generation?: boolean;
credential_recovery_token?: string;
};
type StreamCall = {
@@ -1862,6 +1863,237 @@ describe("WorkflowCopilotChat — credential card wiring", () => {
expect(screen.queryByText(/Credential added/)).toBeNull();
});
+ it.each([
+ ["unknown", /The vault didn't confirm the save/],
+ ["created_not_connected", /Saved as Portal test account, not connected/],
+ ] as const)(
+ "registration Generate and save POSTs only the pause ids and a %s save removes the button",
+ async (generateResult, message) => {
+ sansApiPost.mockImplementation((path: string) =>
+ Promise.resolve(
+ path === "/workflow/copilot/credential-generate"
+ ? { data: { result: generateResult } }
+ : {},
+ ),
+ );
+ await renderChat();
+ await submit("create a test account");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+ expect(streamCalls[0]!.body.supports_credential_generation).toBe(true);
+ await act(async () => {
+ streamCalls[0]!.onMessage(turnStart());
+ streamCalls[0]!.onMessage(
+ credentialFrame({
+ reason: "credential_registration",
+ login_page_urls: ["https://portal.example.com/signup"],
+ registration: {
+ username: "[email protected]",
+ credential_name: "Portal test account",
+ },
+ }),
+ );
+ });
+ await act(async () => {
+ fireEvent.click(
+ screen.getByRole("button", { name: "Generate and save" }),
+ );
+ });
+ const generatePosts = () =>
+ sansApiPost.mock.calls.filter(
+ (call) => call[0] === "/workflow/copilot/credential-generate",
+ );
+ await waitFor(() => expect(generatePosts()).toHaveLength(1));
+ expect(generatePosts()[0]![1]).toEqual({
+ turn_id: "turn-1",
+ workflow_copilot_chat_id: "chat-1",
+ resume_token: "rt-abc",
+ });
+ expect(await screen.findAllByText(message)).not.toHaveLength(0);
+ expect(
+ screen.queryByRole("button", { name: "Generate and save" }),
+ ).toBeNull();
+ expect(credentialResponsePosts()).toHaveLength(0);
+ expect(
+ screen.queryByRole("button", { name: "Skip for now" }) === null,
+ ).toBe(generateResult === "created_not_connected");
+ },
+ );
+
+ const raiseRegistrationCard = async () => {
+ await renderChat();
+ await submit("create a test account");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+ await act(async () => {
+ streamCalls[0]!.onMessage(turnStart());
+ streamCalls[0]!.onMessage(
+ credentialFrame({
+ reason: "credential_registration",
+ login_page_urls: ["https://portal.example.com/signup"],
+ registration: {
+ username: "[email protected]",
+ credential_name: "Portal test account",
+ },
+ }),
+ );
+ });
+ await waitFor(() => expect(credentialsGets().length).toBeGreaterThan(0));
+ };
+
+ it("registration unknown reloads the saved logins so a save that landed late can be picked", async () => {
+ const extendedDeadline = new Date(Date.now() + 600_000).toISOString();
+ sansApiPost.mockImplementation((path: string) =>
+ Promise.resolve(
+ path === "/workflow/copilot/credential-generate"
+ ? {
+ data: {
+ result: "unknown",
+ expires_at: extendedDeadline,
+ },
+ }
+ : {},
+ ),
+ );
+ await raiseRegistrationCard();
+ const listsBefore = credentialsGets().length;
+ credentialsData.current = [
+ {
+ credential_id: "cred-new",
+ name: "Portal test account",
+ tested_url: null,
+ },
+ ];
+ await act(async () => {
+ fireEvent.click(
+ screen.getByRole("button", { name: "Generate and save" }),
+ );
+ });
+
+ await waitFor(() =>
+ expect(credentialsGets().length).toBeGreaterThan(listsBefore),
+ );
+ expect(
+ (
+ await screen.findAllByText(
+ /didn't confirm the save. Check your credentials/,
+ )
+ ).length,
+ ).toBeGreaterThan(0);
+ expect((await screen.findAllByText("Saved logins")).length).toBeGreaterThan(
+ 0,
+ );
+ expect(credentialResponsePosts()).toHaveLength(0);
+ });
+
+ it("registration Generate whose POST fails reads the card back instead of offering a retry", async () => {
+ sansApiPost.mockImplementation((path: string) =>
+ path === "/workflow/copilot/credential-generate"
+ ? Promise.reject(new Error("gateway timeout"))
+ : Promise.resolve({}),
+ );
+ await raiseRegistrationCard();
+ historyResponse.data = {
+ ...historyResponse.data,
+ pending_credential_requests: [
+ credentialFrame({
+ reason: "credential_registration",
+ login_page_urls: ["https://portal.example.com/signup"],
+ registration: {
+ username: "[email protected]",
+ credential_name: "Portal test account",
+ attempted: true,
+ outcome: "unknown",
+ },
+ }),
+ ],
+ };
+ await act(async () => {
+ fireEvent.click(
+ screen.getByRole("button", { name: "Generate and save" }),
+ );
+ });
+
+ expect(
+ (await screen.findAllByText(/The vault didn't confirm the save/)).length,
+ ).toBeGreaterThan(0);
+ expect(
+ apiGet.mock.calls.some(
+ ([path]) => path === "/workflow/copilot/chat-history",
+ ),
+ ).toBe(true);
+ expect(
+ screen.queryByRole("button", { name: "Generate and save" }),
+ ).toBeNull();
+ expect(toastFn).not.toHaveBeenCalled();
+ });
+
+ it("registration Generate whose POST fails keeps reading the card until the save settles", async () => {
+ sansApiPost.mockImplementation((path: string) =>
+ path === "/workflow/copilot/credential-generate"
+ ? Promise.reject(new Error("gateway timeout"))
+ : Promise.resolve({}),
+ );
+ await raiseRegistrationCard();
+ const registrationCard = (
+ registration: Partial<
+ NonNullable<WorkflowCopilotCredentialRequiredUpdate["registration"]>
+ >,
+ expiresAt?: string,
+ ) =>
+ credentialFrame({
+ reason: "credential_registration",
+ login_page_urls: ["https://portal.example.com/signup"],
+ registration: {
+ username: "[email protected]",
+ credential_name: "Portal test account",
+ attempted: true,
+ ...registration,
+ },
+ ...(expiresAt ? { expires_at: expiresAt } : {}),
+ });
+ historyResponse.data = {
+ ...historyResponse.data,
+ pending_credential_requests: [registrationCard({})],
+ };
+ const historyReads = () =>
+ apiGet.mock.calls.filter(
+ ([path]) => path === "/workflow/copilot/chat-history",
+ ).length;
+ vi.useFakeTimers();
+ const readsBefore = historyReads();
+ await act(async () => {
+ fireEvent.click(
+ screen.getByRole("button", { name: "Generate and save" }),
+ );
+ await vi.advanceTimersByTimeAsync(0);
+ });
+
+ expect(historyReads()).toBeGreaterThan(readsBefore);
+ expect(screen.queryByText(/The vault didn't confirm the save/)).toBeNull();
+ expect(
+ screen.queryByRole("button", { name: "Generate and save" }),
+ ).toBeNull();
+
+ const extendedDeadline = new Date(Date.now() + 600_000).toISOString();
+ historyResponse.data = {
+ ...historyResponse.data,
+ pending_credential_requests: [
+ registrationCard({ outcome: "unknown" }, extendedDeadline),
+ ],
+ };
+ await act(async () => vi.advanceTimersByTimeAsync(2_000));
+
+ expect(
+ screen.getAllByText(/The vault didn't confirm the save/).length,
+ ).toBeGreaterThan(0);
+ await act(async () => vi.advanceTimersByTimeAsync(300_001));
+ expect(
+ screen.getAllByText(/The vault didn't confirm the save/).length,
+ ).toBeGreaterThan(0);
+ expect(screen.queryByText("Timed out")).toBeNull();
+ expect(credentialResponsePosts()).toHaveLength(0);
+ expect(toastFn).not.toHaveBeenCalled();
+ });
+
it("connect with an existing matched credential POSTs the credential_id", async () => {
credentialsData.current = [
{
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index c1e848b73..8625f005c 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -123,6 +123,7 @@ import {
WorkflowCopilotWorkflowDraftUpdate,
WorkflowCopilotCodegenProgressUpdate,
WorkflowCopilotCredentialRequiredUpdate,
+ WorkflowCopilotCredentialGenerateResult,
WorkflowCopilotCredentialResponseResult,
WorkflowCopilotCredentialPauseResolvedUpdate,
WorkflowCopilotTitleUpdate,
@@ -378,6 +379,9 @@ const RECOVERY_POLL_STEADY_MS = 30_000;
// up the real reply if the turn finishes after all.
const RECOVERY_POLL_BUDGET_MS = 1_500_000;
const CANONICAL_READ_TIMEOUT_MS = 5_000;
+// Matches the server's MANUAL_SIGN_IN_CLAIM_SECONDS, which bounds a Generate and save.
+const REGISTRATION_SAVE_WINDOW_MS = 120_000;
+const REGISTRATION_SAVE_POLL_MS = 2_000;
const INTERRUPTED_TERMINAL_REASON = "interrupted";
// Only an older backend writes an interrupted row a late reply may still replace.
const isReplaceableInterruptedRow = (
@@ -2548,10 +2552,79 @@ export function WorkflowCopilotChat({
const recoverCredentialTurn = useRef<
(chatId: string, turnId: string) => boolean
>(() => false);
+ const updateRegistration = useCallback(
+ (
+ frame: WorkflowCopilotCredentialRequiredUpdate,
+ update: Partial<
+ NonNullable<WorkflowCopilotCredentialRequiredUpdate["registration"]>
+ >,
+ expiresAt?: string | null,
+ ) => {
+ // The card's Generate button, its countdown, and a reloaded card all read the frame.
+ const apply = (candidate: WorkflowCopilotCredentialRequiredUpdate) =>
+ candidate.resume_token === frame.resume_token && candidate.registration
+ ? {
+ ...candidate,
+ registration: { ...candidate.registration, ...update },
+ expires_at: expiresAt ?? candidate.expires_at,
+ }
+ : candidate;
+ setLivePauseFrame((prev) => (prev ? apply(prev) : prev));
+ setRecoveredPauseFrames((prev) => prev.map(apply));
+ },
+ [],
+ );
+ // A Generate whose answer never arrived may still be saving: read the card back until the save settles.
+ const refreshRegistrationCard = useCallback(
+ async (frame: WorkflowCopilotCredentialRequiredUpdate) => {
+ const stopAt = Date.now() + REGISTRATION_SAVE_WINDOW_MS;
+ const generation = recoveryGeneration.current;
+ const stale = () =>
+ recoveryGeneration.current !== generation ||
+ (workflowCopilotChatIdRef.current !== null &&
+ workflowCopilotChatIdRef.current !== frame.workflow_copilot_chat_id);
+ setCredentialsReloadKey((key) => key + 1);
+ try {
+ const client = await getClient(credentialGetter, "sans-api-v1");
+ for (;;) {
+ if (stale()) return;
+ const response =
+ await readCredentialRecoveryHistory<WorkflowCopilotChatHistoryResponse>(
+ client,
+ workflowPermanentId,
+ {
+ params: {
+ workflow_copilot_chat_id: frame.workflow_copilot_chat_id,
+ },
+ },
+ );
+ const current = response.data.pending_credential_requests?.find(
+ (candidate) => candidate.resume_token === frame.resume_token,
+ );
+ if (stale() || !current?.registration) return;
+ updateRegistration(frame, current.registration, current.expires_at);
+ if (current.registration.outcome) {
+ setCredentialsReloadKey((key) => key + 1);
+ return;
+ }
+ if (Date.now() >= stopAt) return;
+ await new Promise((resolve) =>
+ setTimeout(resolve, REGISTRATION_SAVE_POLL_MS),
+ );
+ }
+ } catch (error) {
+ console.error(
+ "Failed to refresh the credential card:",
+ error instanceof Error ? error.message : String(error),
+ );
+ }
+ },
+ [credentialGetter, updateRegistration, workflowPermanentId],
+ );
const respondToCredentialPause = useCallback(
async (
frame: WorkflowCopilotCredentialRequiredUpdate,
- action: "connected" | "skip" | "signing_in" | "signed_in",
+ action: "connected" | "skip" | "signing_in" | "signed_in" | "generate",
credentialId?: string,
name?: string,
): Promise<WorkflowCopilotCredentialResponseResult | undefined> => {
@@ -2573,22 +2646,40 @@ export function WorkflowCopilotChat({
// Copilot routes live on base_router (no /api/v1 prefix), like cancel.
const client = await getClient(credentialGetter, "sans-api-v1");
if (!isCopilotTurnCurrent(reservation)) return;
- const response =
- await client.post<WorkflowCopilotCredentialResponseResult>(
- "/workflow/copilot/credential-response",
- {
- turn_id: frame.turn_id,
- workflow_copilot_chat_id: frame.workflow_copilot_chat_id,
- resume_token: frame.resume_token,
- action,
- credential_id: action === "connected" ? credentialId : undefined,
- },
- );
- // A backend that predates sign-in answers 204 with no body.
- const result: WorkflowCopilotCredentialResponseResult = response.data
- ?.result
- ? response.data
- : { result: "accepted" };
+ const pauseIds = {
+ turn_id: frame.turn_id,
+ workflow_copilot_chat_id: frame.workflow_copilot_chat_id,
+ resume_token: frame.resume_token,
+ };
+ let generated: WorkflowCopilotCredentialGenerateResult | null = null;
+ let result: WorkflowCopilotCredentialResponseResult = {
+ result: "accepted",
+ };
+ if (action === "generate") {
+ // The server allows one attempt per card, so Generate goes away as soon as it is sent.
+ updateRegistration(frame, { attempted: true });
+ generated = (
+ await client.post<WorkflowCopilotCredentialGenerateResult>(
+ "/workflow/copilot/credential-generate",
+ pauseIds,
+ )
+ ).data;
+ // Any answer, even an unconfirmed one, can mean a new saved login the picker should list.
+ setCredentialsReloadKey((key) => key + 1);
+ } else {
+ const response =
+ await client.post<WorkflowCopilotCredentialResponseResult>(
+ "/workflow/copilot/credential-response",
+ {
+ ...pauseIds,
+ action,
+ credential_id:
+ action === "connected" ? credentialId : undefined,
+ },
+ );
+ // A backend that predates sign-in answers 204 with no body.
+ if (response.data?.result) result = response.data;
+ }
if (
!isCopilotTurnCurrent(reservation) ||
recoveryGeneration.current !== generation ||
@@ -2604,6 +2695,25 @@ export function WorkflowCopilotChat({
frame.turn_id,
);
}
+ if (generated && generated.result !== "connected") {
+ updateRegistration(
+ frame,
+ { outcome: generated.result },
+ generated.expires_at,
+ );
+ // The pause ended before the save landed, so its resume token answers nothing now.
+ if (generated.result === "created_not_connected") {
+ setPauseCardResolutions((prev) =>
+ prev[frame.resume_token]
+ ? prev
+ : withCappedResolution(prev, frame.resume_token, {
+ outcome: "timeout",
+ name: generated.name ?? undefined,
+ }),
+ );
+ }
+ return result;
+ }
if (
action === "signing_in" ||
result.result === "no_sign_in_found" ||
@@ -2611,8 +2721,13 @@ export function WorkflowCopilotChat({
) {
return result;
}
- const resolution: CredentialResolution =
- action === "connected"
+ const resolution: CredentialResolution = generated
+ ? {
+ outcome: "connected",
+ credentialId: generated.credential_id ?? undefined,
+ name: generated.name ?? undefined,
+ }
+ : action === "connected"
? { outcome: "connected", credentialId, name }
: action === "signed_in"
? { outcome: "signed_in" }
@@ -2643,6 +2758,11 @@ export function WorkflowCopilotChat({
error,
frame.workflow_copilot_chat_id,
);
+ // Generate is never retried: the save may have gone through, so read the card back instead.
+ if (action === "generate") {
+ void refreshRegistrationCard(frame);
+ return;
+ }
toast({
title: "Couldn't send your credential response",
description: "Please try again.",
@@ -2652,7 +2772,13 @@ export function WorkflowCopilotChat({
credentialResponseInFlight.current = false;
}
},
- [credentialGetter, isCopilotTurnCurrent, logCopilotRequestFailure],
+ [
+ credentialGetter,
+ isCopilotTurnCurrent,
+ logCopilotRequestFailure,
+ refreshRegistrationCard,
+ updateRegistration,
+ ],
);
const startManualSignIn = useCallback(
async (frame: WorkflowCopilotCredentialRequiredUpdate) => {
@@ -8416,6 +8542,7 @@ export function WorkflowCopilotChat({
selected_block_label: readSelectedBlockLabel(),
keep_pending_proposal: Boolean(pendingProposalTurnId),
supports_credential_pause: true,
+ supports_credential_generation: true,
supports_credential_pause_recovery: Boolean(
credentialRecoveryToken,
),
@@ -9462,6 +9589,7 @@ export function WorkflowCopilotChat({
: openCredentialModal(frame, frame.turn_id)
}
onSkip={() => void respondToCredentialPause(frame, "skip")}
+ onGenerate={() => void respondToCredentialPause(frame, "generate")}
signIn={
frame.sign_in_browser_session_id &&
frame.sign_in_browser_session_id === liveBrowserSessionId
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.fixtures.ts b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.fixtures.ts
index d34d72432..283ce728b 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.fixtures.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.fixtures.ts
@@ -60,6 +60,15 @@ export const CREDENTIAL_REQUIRED_FRAME_BY_REASON = {
message: "The site rejected the saved one-time code.",
credential_refs: ["cred_hn"],
}),
+ credential_registration: buildCredentialRequiredFrame({
+ reason: "credential_registration",
+ message: "Create the test account you asked for.",
+ login_page_urls: ["https://portal.example.com/signup"],
+ registration: {
+ username: "[email protected]",
+ credential_name: "Portal test account",
+ },
+ }),
} as const;
// No dynamic ask text; only a richer, timed pause signal supplies one.
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx
index 802c1b3a0..b826d4131 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx
@@ -429,6 +429,69 @@ describe("CredentialCard sign in myself", () => {
});
});
+describe("CredentialCard registration", () => {
+ const registrationFrame =
+ CREDENTIAL_REQUIRED_FRAME_BY_REASON.credential_registration;
+
+ it("shows the exact sign-up page, username and saved name, and Generate and save answers it", () => {
+ const onGenerate = vi.fn();
+ const onSkip = vi.fn();
+ render(
+ <CredentialCard
+ frame={registrationFrame}
+ mode="inline-pause"
+ onConnect={vi.fn()}
+ onSkip={onSkip}
+ onGenerate={onGenerate}
+ />,
+ );
+ expect(
+ screen.getByText("Create a login for https://portal.example.com"),
+ ).toBeTruthy();
+ expect(
+ screen.getByText("Sign-up page: https://portal.example.com/signup"),
+ ).toBeTruthy();
+ expect(screen.getByText("Username: [email protected]")).toBeTruthy();
+ expect(screen.getByText("Saved as: Portal test account")).toBeTruthy();
+ fireEvent.click(screen.getByRole("button", { name: "Generate and save" }));
+ expect(onGenerate).toHaveBeenCalledTimes(1);
+ expect(onSkip).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ["rejected", /Nothing was saved/],
+ ["unknown", /The vault didn't confirm the save/],
+ ["created_not_connected", /Saved as Portal test account, not connected/],
+ ] as const)(
+ "after a %s attempt, hides Generate and says what happened",
+ (outcome, message) => {
+ render(
+ <CredentialCard
+ frame={{
+ ...registrationFrame,
+ registration: {
+ ...registrationFrame.registration!,
+ attempted: true,
+ outcome,
+ },
+ }}
+ mode="inline-pause"
+ onConnect={vi.fn()}
+ onSkip={vi.fn()}
+ onGenerate={vi.fn()}
+ />,
+ );
+ expect(
+ screen.queryByRole("button", { name: "Generate and save" }),
+ ).toBeNull();
+ expect(screen.getAllByText(message)).not.toHaveLength(0);
+ expect(
+ screen.getByRole("button", { name: "Connect credential" }),
+ ).toBeTruthy();
+ },
+ );
+});
+
describe("CredentialCard terminal org-credential picker", () => {
it("fetches the org credentials and renders them as a picker, in API order", async () => {
credsData.current = [
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx
index be8e194c4..bc87576ac 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx
@@ -61,7 +61,16 @@ export type CredentialRequiredReason =
| "assistant_directed"
| "missing_credential_run_failure"
| "credential_missing_totp"
- | "credential_rejected_by_site";
+ | "credential_rejected_by_site"
+ | "credential_registration";
+
+// What Generate and save would store. Never carries a password; the server generates it.
+export interface CredentialRegistrationDetails {
+ username: string;
+ credential_name: string;
+ attempted?: boolean;
+ outcome?: "rejected" | "unknown" | "created_not_connected" | null;
+}
export interface CredentialRequiredFrame {
type: "credential_required";
@@ -75,6 +84,7 @@ export interface CredentialRequiredFrame {
timeout_seconds?: number;
expires_at?: string;
signing_in?: boolean;
+ registration?: CredentialRegistrationDetails | null;
timestamp?: string;
}
@@ -135,6 +145,8 @@ export interface CredentialCardProps {
tray?: AttentionTrayPresentation;
// Offered only when the card's browser is the one on screen, so the user signs in where the agent looks.
signIn?: ManualSignInOffer;
+ // Registration cards only: saves a server-generated password under the frame's username and name.
+ onGenerate?: () => void;
}
export interface ManualSignInOffer {
@@ -173,6 +185,8 @@ export const CREDENTIAL_WHY_LINE_BY_REASON: Record<
"This saved login has no 2FA method, so the workflow can't pass the verification step. Add one in the credential editor — codes never go through chat.",
credential_rejected_by_site:
"Update the saved password or one-time code here and I'll try the sign-in again.",
+ credential_registration:
+ "Generate and save creates a strong password and stores it in your credentials — it never appears in chat. Saving it does not create the account; I'll still submit the sign-up form.",
};
// Mirrors the credentials route: it caps `search` at 200 characters and pages at 100. A longer term
@@ -774,6 +788,7 @@ function CredentialAskCard({
canChange = false,
tray,
signIn,
+ onGenerate,
}: Readonly<CredentialCardProps>) {
// Terminal mode never expires by design: its signal carries no timeout/expiry
// semantics at all, so there is nothing to compare "now" against. Only a
@@ -925,7 +940,15 @@ function CredentialAskCard({
/>
);
case "timeout":
- return <ResolvedCredentialCard tone="warn" {...TIMEOUT_OUTCOME} />;
+ return frame.registration?.outcome === "created_not_connected" ? (
+ <ResolvedCredentialCard
+ tone="warn"
+ title={`Saved as ${frame.registration.credential_name}, not connected`}
+ detail="The request ended before this login was connected. It is on the Credentials page."
+ />
+ ) : (
+ <ResolvedCredentialCard tone="warn" {...TIMEOUT_OUTCOME} />
+ );
case "signed_in":
return (
<ResolvedCredentialCard
@@ -1049,11 +1072,41 @@ function CredentialAskCard({
}
const site = siteFromLoginPageUrls(frame.login_page_urls);
+ const registration = frame.registration ?? null;
+ const offerGenerate = Boolean(
+ registration && onGenerate && !registration.attempted,
+ );
+ const registrationLines = registration
+ ? [
+ <span key="destination" className="text-[11px] leading-relaxed">
+ Sign-up page: {frame.login_page_urls?.[0] ?? site}
+ </span>,
+ <span key="username" className="text-[11px] leading-relaxed">
+ Username: {registration.username}
+ </span>,
+ <span key="saved-as" className="text-[11px] leading-relaxed">
+ Saved as: {registration.credential_name}
+ </span>,
+ ...(registration.outcome
+ ? [
+ <span key="outcome" className="text-[11px] font-medium">
+ {registration.outcome === "rejected"
+ ? "Nothing was saved — the credential couldn't be created. Pick or add a login instead."
+ : registration.outcome === "created_not_connected"
+ ? `Saved as ${registration.credential_name}, not connected.`
+ : "The vault didn't confirm the save. Check your credentials before adding another."}
+ </span>,
+ ]
+ : []),
+ ]
+ : [];
const connectButton = (
<Button
type="button"
size="sm"
- variant={signIn && frame.signing_in ? "outline" : "default"}
+ variant={
+ (signIn && frame.signing_in) || offerGenerate ? "outline" : "default"
+ }
disabled={disabled}
onClick={() => onConnect(undefined)}
data-tour="credential-connect"
@@ -1133,7 +1186,11 @@ function CredentialAskCard({
tray={tray}
rootRef={rootRef}
message={frame.message}
- title={`Copilot needs to sign in to ${site}`}
+ title={
+ registration
+ ? `Create a login for ${site}`
+ : `Copilot needs to sign in to ${site}`
+ }
countdown={
countdownActive ? (
<PauseCountdown remainingMs={remainingMs} expired={expired} />
@@ -1146,6 +1203,7 @@ function CredentialAskCard({
>
{CREDENTIAL_WHY_LINE_BY_REASON[frame.reason] ?? SIGN_IN_WHY_LINE}
</span>,
+ ...registrationLines,
...(mode === "terminal"
? [
<span
@@ -1159,6 +1217,16 @@ function CredentialAskCard({
]}
footer={
<>
+ {offerGenerate ? (
+ <Button
+ type="button"
+ size="sm"
+ disabled={disabled}
+ onClick={() => onGenerate?.()}
+ >
+ Generate and save
+ </Button>
+ ) : null}
{connectButton}
{signIn ? (
<Button
diff --git a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
index 96df2f1a5..16c34f620 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/copilot/workflowCopilotTypes.ts
@@ -168,6 +168,7 @@ export interface WorkflowCopilotChatRequest {
// Opt-in: only clients that can render the credential_required frame set
// this, so the backend never pauses a turn a client would silently drop.
supports_credential_pause?: boolean;
+ supports_credential_generation?: boolean;
credential_recovery_token?: string;
supports_credential_pause_recovery?: boolean;
supports_question_tool?: boolean;
@@ -435,6 +436,13 @@ export interface WorkflowCopilotCredentialRequiredUpdate {
// The live browser the user may sign in to themselves; absent when the card does not offer it.
sign_in_browser_session_id?: string | null;
signing_in?: boolean;
+ // Present on a card that offers Generate and save; never carries a password.
+ registration?: {
+ username: string;
+ credential_name: string;
+ attempted?: boolean;
+ outcome?: "rejected" | "unknown" | "created_not_connected" | null;
+ } | null;
timestamp: string;
}
@@ -445,6 +453,15 @@ export interface WorkflowCopilotCredentialResponseResult {
browser_profile_id?: string | null;
}
+export interface WorkflowCopilotCredentialGenerateResult {
+ result: "connected" | "rejected" | "unknown" | "created_not_connected";
+ credential_id?: string | null;
+ name?: string | null;
+ username?: string | null;
+ // The reopened card's new deadline after "rejected" or "unknown".
+ expires_at?: string | null;
+}
+
export interface WorkflowCopilotCredentialPauseResolvedUpdate {
type: "credential_pause_resolved";
turn_id: string;
diff --git a/skyvern/forge/log_redaction.py b/skyvern/forge/log_redaction.py
index d441e2aac..e102c0381 100644
--- a/skyvern/forge/log_redaction.py
+++ b/skyvern/forge/log_redaction.py
@@ -42,6 +42,7 @@ SENSITIVE_HEADERS = {
"x-marketplace-api-key",
"x-posthog-attribution",
"x-copilot-credential-recovery-token",
+ "stripe-signature",
}
# Exact field names that are always redacted. Use a set for O(1) lookup
@@ -79,6 +80,11 @@ SENSITIVE_FIELDS: set[str] = {
"client_secret",
"access_token",
"refresh_token",
+ # Masks only billing_details, payment_method_details and receipt_email, wherever they nest. Card blocks and
+ # invoice or checkout customer fields are not covered; the webhook handler's identifier-only log lines protect them.
+ "billing_details",
+ "payment_method_details",
+ "receipt_email",
} | SENSITIVE_HEADERS
_PROXY_OBSERVABILITY_FIELDS = {field.value: field for field in ProxyObservabilityField}
diff --git a/skyvern/forge/request_logging.py b/skyvern/forge/request_logging.py
index b0bdb67ce..630874e31 100644
--- a/skyvern/forge/request_logging.py
+++ b/skyvern/forge/request_logging.py
@@ -57,6 +57,8 @@ _SENSITIVE_ENDPOINTS = {
"POST /v1/workflow/copilot/question-response",
"POST /v1/workflow/copilot/credential-response",
"POST /v1/workflow/copilot/convert-yaml-to-blocks",
+ # Payment-provider events carry customer billing details (name, email, address, card metadata).
+ "POST /api/v1/stripe_webhook",
}
_SENSITIVE_ENDPOINT_PATTERNS = (
re.compile(r"^(?:POST|PUT) /(?:api/)?v1/credentials(?:/.*)?$"),
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index c99d56342..7a9588428 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -124,6 +124,8 @@ from skyvern.forge.sdk.copilot.context import (
from skyvern.forge.sdk.copilot.credential_pause import credential_recovery_token_digest
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.enforcement import (
+ FINAL_REPLY_OBSERVATION,
+ RAW_SECRET_REPLY_WITHHELD_OBSERVATION,
CopilotNonRetriableNavError,
_elapsed_run_seconds,
artifact_health_blocked,
@@ -4359,6 +4361,7 @@ async def _run_agent_loop_with_surface(
is_fallback: bool = False,
session: Session | None = None,
final_reply: bool = False,
+ final_reply_observation: str = FINAL_REPLY_OBSERVATION,
) -> Any:
# No model owns the attempt until setup completes and enforcement is ready to
# enter the model loop. This also clears a prior model before fallback setup.
@@ -4429,6 +4432,7 @@ async def _run_agent_loop_with_surface(
session=session,
hooks=FinalReplyRunHooks(ctx),
run_config=run_config,
+ observation=final_reply_observation,
)
else:
result = await run_with_enforcement(
@@ -4577,6 +4581,44 @@ def _build_request_policy_clarification_result(
)
+PENDING_PROPOSAL_TEST_NOT_RESTORED_REPLY = (
+ "Nothing was run: the pending Copilot proposal changed before this test started, so it would not have "
+ "run the proposal as offered. Reload and test the current proposal."
+)
+
+
+def _build_pending_proposal_test_refusal_result(
+ ctx: CopilotContext,
+ prior_global_llm_context: str | None,
+ prior_workflow_yaml: str | None,
+) -> AgentResult:
+ final_text, outcome = apply_repeated_reply_guard(
+ final_text=PENDING_PROPOSAL_TEST_NOT_RESTORED_REPLY,
+ attempted_kind=ResponseKind.REFUSE,
+ blocked_signatures=list(ctx.blocked_reply_signatures),
+ reason_code="pending_proposal_test_not_restored",
+ )
+ return _make_agent_result(
+ ctx,
+ user_response=final_text,
+ updated_workflow=None,
+ global_llm_context=prior_global_llm_context,
+ workflow_yaml=prior_workflow_yaml or None,
+ workflow_was_persisted=False,
+ clear_proposed_workflow=False,
+ proposal_disposition="no_proposal",
+ turn_outcome=outcome,
+ turn_id=ctx.turn_id,
+ narrative_summary=ctx.narrative_summary,
+ narrative_payload=_build_narrative_payload(
+ ctx,
+ terminal="response",
+ terminal_message=final_text,
+ narrative_summary=ctx.narrative_summary,
+ ),
+ )
+
+
def _agent_output_to_text(agent_output: Any) -> str:
if isinstance(agent_output, str):
return agent_output
@@ -5274,6 +5316,7 @@ async def _run_copilot_turn_impl(
selected_block_label=getattr(chat_request, "selected_block_label", None),
client_supports_credential_pause=getattr(chat_request, "supports_credential_pause", False),
client_supports_credential_pause_recovery=chat_request.supports_credential_pause_recovery,
+ client_supports_credential_generation=chat_request.supports_credential_generation,
credential_recovery_token_digest=credential_recovery_token_digest(
chat_request.credential_recovery_token.get_secret_value()
if chat_request.credential_recovery_token
@@ -5505,6 +5548,12 @@ async def _run_copilot_turn_impl(
get_skyvern_mcp_alias_map,
)
+ if chat_request.product_action == "test_end_to_end" and ctx.staged_workflow is None:
+ # Without this gate the route-time proposal bytes would be staged, and the run would be created and then
+ # refused at bind.
+ LOG.info("copilot_pending_proposal_test_refused", reason="proposal_not_restored")
+ return _build_pending_proposal_test_refusal_result(ctx, global_llm_context, chat_request.workflow_yaml)
+
validated_browser_session_id = await _resolve_live_browser_session_id(chat_request, organization_id, ctx)
ctx.browser_session_id = validated_browser_session_id
@@ -5759,6 +5808,7 @@ async def _run_copilot_turn_impl(
*,
is_fallback: bool,
final_reply: bool = False,
+ final_reply_observation: str = FINAL_REPLY_OBSERVATION,
) -> RunResultStreaming:
attempt = await _run_agent_loop_with_surface(
ctx=ctx,
@@ -5778,6 +5828,7 @@ async def _run_copilot_turn_impl(
is_fallback=is_fallback,
session=model_session,
final_reply=final_reply,
+ final_reply_observation=final_reply_observation,
)
return attempt
@@ -5792,6 +5843,38 @@ async def _run_copilot_turn_impl(
evaluated_reason_codes=_output_policy_reason_codes_from_guardrail_exception(exc),
)
+ async def _reask_after_withheld_secret_reply(exc: OutputGuardrailTripwireTriggered) -> AgentResult:
+ # The SDK saves a final reply to the session only after its output guardrail passes, so the
+ # withheld text is not in the history the re-ask reads. A second hit keeps the refusal.
+ if _output_policy_verdict_from_guardrail_exception(exc).reason_codes != [OutputPolicyReason.RAW_SECRET_LEAK]:
+ return _output_policy_blocked(exc)
+ LOG.info("copilot_raw_secret_reply_reasked")
+ try:
+ reply = await _run_attempt(
+ model_name,
+ run_config,
+ llm_key,
+ is_fallback=False,
+ final_reply=True,
+ final_reply_observation=RAW_SECRET_REPLY_WITHHELD_OBSERVATION,
+ )
+ return await _translate_to_agent_result(
+ reply,
+ ctx,
+ global_llm_context,
+ chat_request,
+ organization_id,
+ final_reply=True,
+ )
+ except asyncio.CancelledError:
+ LOG.info("Copilot run cancelled")
+ return _build_cancelled_exit_result(ctx, global_llm_context)
+ except OutputGuardrailTripwireTriggered as reask_exc:
+ return _output_policy_blocked(reask_exc)
+ except Exception as reask_error:
+ LOG.warning("copilot_raw_secret_reply_reask_failed", error_type=type(reask_error).__name__)
+ return _output_policy_blocked(exc)
+
try:
from skyvern.forge.sdk.copilot.session_factory import create_copilot_session
@@ -5870,7 +5953,7 @@ async def _run_copilot_turn_impl(
ctx=ctx,
)
except OutputGuardrailTripwireTriggered as exc:
- return _output_policy_blocked(exc)
+ return await _reask_after_withheld_secret_reply(exc)
except CopilotTurnHalt as exc:
LOG.info(
"Copilot run stopped after typed turn halt",
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 246f748f9..0b3fc05a9 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -1247,6 +1247,7 @@ class CopilotContext(AgentContext):
last_run_skipped_unbound_credentials: bool = False
client_supports_credential_pause: bool = False
client_supports_credential_pause_recovery: bool = False
+ client_supports_credential_generation: bool = False
credential_recovery_token_digest: str | None = field(default=None, repr=False)
credential_recovery_armed: bool = False
credential_pause_used: bool = False
@@ -1259,6 +1260,9 @@ class CopilotContext(AgentContext):
credential_pause_reaskable_by_run: bool = False
copilot_credential_pause_seconds: float = 0.0
credential_pause_outcome: str | None = None
+ credential_registration_outcome: Literal["rejected", "unknown"] | None = None
+ # Generate and save created, or may have created, a credential; a second generate card could mint a duplicate.
+ credential_generation_spent: bool = False
credential_pause_connected_credential_id: str | None = None
credential_pause_anchor_tool_call_id: str | None = None
# Set while a ``request_credential`` ask is open, so tool calls issued alongside it in the same
@@ -1433,6 +1437,9 @@ class CopilotContext(AgentContext):
proposal_revision: int | None = None
proposal_canonical_fingerprint: str | None = None
proposal_workflow_run_id: str | None = None
+ # A restored candidate whose request private settings differ from its own; its stored bytes cannot vouch
+ # for what a test under this token would run, so binding a run to it is refused.
+ settings_diverged_proposal_token: tuple[str, int] | None = None
# The chat row's setting, not the turn's commit decision: the route can still refuse to apply a
# staged draft at turn end. None on entrypoints that load no chat row.
auto_accept: bool | None = None
diff --git a/skyvern/forge/sdk/copilot/credential_generation.py b/skyvern/forge/sdk/copilot/credential_generation.py
new file mode 100644
index 000000000..14d2a2d50
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/credential_generation.py
@@ -0,0 +1,27 @@
+import secrets
+import string
+from typing import Literal
+
+REGISTRATION_PASSWORD_MIN_LENGTH = 24
+REGISTRATION_PASSWORD_MAX_LENGTH = 128
+SAFE_SYMBOLS = "!#$%*+-=?@^_~"
+
+RegistrationCharset = Literal["alphanumeric", "alphanumeric_symbols"]
+
+
+def _classes(charset: RegistrationCharset) -> list[str]:
+ classes = [string.ascii_lowercase, string.ascii_uppercase, string.digits]
+ return classes + [SAFE_SYMBOLS] if charset == "alphanumeric_symbols" else classes
+
+
+def generate_registration_password(length: int, charset: RegistrationCharset) -> str:
+ # 24 characters from the 62-character alphabet already carry 142 bits; redrawing until every class
+ # appears costs well under one bit.
+ if not REGISTRATION_PASSWORD_MIN_LENGTH <= length <= REGISTRATION_PASSWORD_MAX_LENGTH:
+ raise ValueError("password length out of range")
+ classes = _classes(charset)
+ alphabet = "".join(classes)
+ while True:
+ password = "".join(secrets.choice(alphabet) for _ in range(length))
+ if all(any(char in chars for char in password) for chars in classes):
+ return password
diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.py
index 875ef879c..54202c91c 100644
--- a/skyvern/forge/sdk/copilot/credential_pause.py
+++ b/skyvern/forge/sdk/copilot/credential_pause.py
@@ -48,6 +48,7 @@ from skyvern.forge.sdk.schemas.credentials import Credential
from skyvern.forge.sdk.schemas.workflow_copilot import (
CredentialPauseResolvedOutcome,
WorkflowCopilotCredentialPauseResolvedUpdate,
+ WorkflowCopilotCredentialRegistration,
WorkflowCopilotCredentialRequiredUpdate,
WorkflowCopilotStreamMessageType,
)
@@ -85,7 +86,7 @@ def _credential_pause_lock_key(organization_id: str, chat_id: str, turn_id: str)
# A Done claim reserves the pause while the profile is saved outside the lock; the save itself is bounded
-# tighter so the claim always outlives it.
+# tighter so the claim always outlives it. A Generate and save claim reuses the same window.
MANUAL_SIGN_IN_CLAIM_SECONDS = 120
MANUAL_SIGN_IN_SAVE_TIMEOUT_SECONDS = 90
@@ -144,6 +145,8 @@ class _ActivePauseRecord:
manual_sign_in: ManualSignIn | None = None
claim_id: str | None = None
claim_deadline: datetime | None = None
+ registration: WorkflowCopilotCredentialRegistration | None = None
+ generated_credential_id: str | None = None
def live_until(self) -> datetime | None:
if self.status == "pending":
@@ -174,6 +177,8 @@ def _encode_record(record: _ActivePauseRecord) -> str:
),
"claim_id": record.claim_id,
"claim_deadline": record.claim_deadline.isoformat() if record.claim_deadline else None,
+ "registration": record.registration.model_dump(mode="json") if record.registration else None,
+ "generated_credential_id": record.generated_credential_id,
}
)
@@ -186,6 +191,7 @@ def _encode_active_pause(
card: WorkflowCopilotCredentialRequiredUpdate | None = None,
recovery_token_digest: str | None = None,
manual_sign_in: ManualSignIn | None = None,
+ registration: WorkflowCopilotCredentialRegistration | None = None,
) -> str:
return _encode_record(
_ActivePauseRecord(
@@ -195,6 +201,7 @@ def _encode_active_pause(
card=card,
recovery_token_digest=recovery_token_digest,
manual_sign_in=manual_sign_in,
+ registration=registration,
)
)
@@ -245,8 +252,17 @@ def _decode_active_pause(raw: Any) -> _ActivePauseRecord | None:
card = WorkflowCopilotCredentialRequiredUpdate.model_validate(data["card"]) if data.get("card") else None
except ValidationError:
card = None
+ try:
+ registration = (
+ WorkflowCopilotCredentialRegistration.model_validate(data["registration"])
+ if data.get("registration")
+ else None
+ )
+ except ValidationError:
+ registration = None
digest = data.get("recovery_token_digest")
claim_id = data.get("claim_id")
+ generated_credential_id = data.get("generated_credential_id")
return _ActivePauseRecord(
resume_token=token,
status=record_status,
@@ -256,6 +272,8 @@ def _decode_active_pause(raw: Any) -> _ActivePauseRecord | None:
manual_sign_in=_decode_manual_sign_in(data.get("manual_sign_in")),
claim_id=claim_id if isinstance(claim_id, str) else None,
claim_deadline=_decode_datetime(data.get("claim_deadline")),
+ registration=registration,
+ generated_credential_id=generated_credential_id if isinstance(generated_credential_id, str) else None,
)
@@ -355,7 +373,8 @@ async def pending_credential_requests(
and card.resume_token == record.resume_token
and card.expires_at == record.expires_at
):
- cards.append(card)
+ registration = record.registration
+ cards.append(card.model_copy(update={"registration": registration}) if registration else card)
return cards
@@ -431,7 +450,13 @@ async def resolve_credential_pause(
async with cache.get_lock(lock_key):
record = _validate_pending_pause(_decode_active_pause(await cache.get(active_key)), resume_token)
ttl = _credential_pause_record_ttl(settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS)
- await cache.set(active_key, _encode_active_pause(record.resume_token, record.expires_at, consumed=True), ex=ttl)
+ await cache.set(
+ active_key,
+ _encode_active_pause(
+ record.resume_token, record.expires_at, consumed=True, registration=record.registration
+ ),
+ ex=ttl,
+ )
await cache.set(
credential_response_cache_key(organization_id, workflow_copilot_chat_id, turn_id),
encode_credential_response(action, credential_id, record.resume_token),
@@ -452,6 +477,7 @@ class CredentialPauseResolution:
action: Literal["connected", "skip", "signed_in"]
credential: Credential | None = None
signed_in: SignedInProfile | None = None
+ generated: bool = False
def encode_credential_response(
@@ -531,15 +557,20 @@ async def start_manual_sign_in(
)
if sign_in.started:
return record.expires_at
- expires_at = max(
- record.expires_at,
- datetime.now(timezone.utc) + timedelta(seconds=settings.WORKFLOW_COPILOT_MANUAL_SIGN_IN_TIMEOUT_SECONDS),
+ record = _extend_pause(
+ replace(record, manual_sign_in=replace(sign_in, started=True)),
+ settings.WORKFLOW_COPILOT_MANUAL_SIGN_IN_TIMEOUT_SECONDS,
+ signing_in=True,
)
- # The recovered card must carry the record's deadline, or pending_credential_requests drops it.
- card = record.card.model_copy(update={"expires_at": expires_at, "signing_in": True}) if record.card else None
- record = replace(record, expires_at=expires_at, card=card, manual_sign_in=replace(sign_in, started=True))
await cache.set(active_key, _encode_record(record), ex=_longest_credential_pause_record_ttl())
- return expires_at
+ return record.expires_at
+
+
+def _extend_pause(record: _ActivePauseRecord, seconds: int, **card_update: bool) -> _ActivePauseRecord:
+ expires_at = max(record.expires_at, datetime.now(timezone.utc) + timedelta(seconds=seconds))
+ # The recovered card must carry the record's deadline, or pending_credential_requests drops it.
+ card = record.card.model_copy(update={"expires_at": expires_at, **card_update}) if record.card else None
+ return replace(record, expires_at=expires_at, card=card)
@dataclass(frozen=True)
@@ -608,6 +639,87 @@ async def finish_manual_sign_in(
return True
+@dataclass(frozen=True)
+class CredentialGenerationClaim:
+ claim_id: str
+ registration: WorkflowCopilotCredentialRegistration
+ deadline: datetime
+
+
+async def claim_credential_generation(
+ cache: BaseCache,
+ *,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ resume_token: str,
+) -> CredentialGenerationClaim:
+ """Reserve a registration card's one Generate and save while the credential is created outside the lock."""
+ active_key = credential_pause_active_key(organization_id, workflow_copilot_chat_id, turn_id)
+ async with cache.get_lock(_credential_pause_lock_key(organization_id, workflow_copilot_chat_id, turn_id)):
+ record = _validate_pending_pause(_decode_active_pause(await cache.get(active_key)), resume_token)
+ registration = record.registration
+ if registration is None:
+ raise CredentialPauseRejection(
+ status_code=HTTPStatus.CONFLICT,
+ detail="This credential card does not offer generating a credential",
+ )
+ if registration.attempted:
+ raise CredentialPauseRejection(
+ status_code=HTTPStatus.CONFLICT,
+ detail="This credential card already tried to generate a credential",
+ )
+ claim_id = secrets.token_hex(16)
+ registration = registration.model_copy(update={"attempted": True})
+ deadline = datetime.now(timezone.utc) + timedelta(seconds=MANUAL_SIGN_IN_CLAIM_SECONDS)
+ claimed = replace(
+ record, status="resolving", claim_id=claim_id, claim_deadline=deadline, registration=registration
+ )
+ await cache.set(active_key, _encode_record(claimed), ex=_longest_credential_pause_record_ttl())
+ return CredentialGenerationClaim(claim_id=claim_id, registration=registration, deadline=deadline)
+
+
+async def finish_credential_generation(
+ cache: BaseCache,
+ *,
+ organization_id: str,
+ workflow_copilot_chat_id: str,
+ turn_id: str,
+ claim: CredentialGenerationClaim,
+ credential_id: str | None,
+ outcome: Literal["rejected", "unknown"] | None = None,
+) -> datetime | None:
+ """Connect the created credential, or reopen the card without Generate and with a fresh deadline.
+
+ Returns the pause's deadline, or None when the claim was lost."""
+ active_key = credential_pause_active_key(organization_id, workflow_copilot_chat_id, turn_id)
+ if credential_id is not None:
+ await _record_turn_resumed(organization_id, workflow_copilot_chat_id, turn_id)
+ async with cache.get_lock(_credential_pause_lock_key(organization_id, workflow_copilot_chat_id, turn_id)):
+ record = _decode_active_pause(await cache.get(active_key))
+ if record is None or record.status != "resolving" or record.claim_id != claim.claim_id:
+ return None
+ ttl = _longest_credential_pause_record_ttl()
+ if credential_id is None:
+ registration = claim.registration.model_copy(update={"outcome": outcome})
+ reopened = _extend_pause(
+ replace(record, status="pending", claim_id=None, claim_deadline=None, registration=registration),
+ settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS,
+ )
+ await cache.set(active_key, _encode_record(reopened), ex=ttl)
+ return reopened.expires_at
+ consumed = _ActivePauseRecord(
+ record.resume_token, "consumed", record.expires_at, generated_credential_id=credential_id
+ )
+ await cache.set(active_key, _encode_record(consumed), ex=ttl)
+ await cache.set(
+ credential_response_cache_key(organization_id, workflow_copilot_chat_id, turn_id),
+ encode_credential_response("connected", credential_id, record.resume_token),
+ ex=ttl,
+ )
+ return record.expires_at
+
+
def credential_pause_reason(ctx: Any) -> str | None:
"""Typed-signal-only detector for a mid-build credential ask.
@@ -880,6 +992,7 @@ async def _run_credential_pause(
admit_connected: Callable[[Credential], Awaitable[bool]] | None = None,
allow_second_ask: bool = False,
anchor_tool_call_id: str | None = None,
+ registration: WorkflowCopilotCredentialRegistration | None = None,
) -> CredentialPauseResolution | None:
"""Send the credential card and wait for the user's decision.
@@ -889,6 +1002,7 @@ async def _run_credential_pause(
Recovery-capable clients restore the same active card through chat history.
"""
update_ask = update_credential_id is not None
+ ctx.credential_registration_outcome = None
if not credential_pause_transport_ready(ctx, copilot_config, allow_second_ask=update_ask or allow_second_ask):
return None
# Latch before async checks so a declined transport cannot trigger another pause. Only the pick ask
@@ -931,9 +1045,9 @@ async def _run_credential_pause(
turn_id = ctx.turn_id or ""
resume_token = _new_resume_token()
expires_at = now + timedelta(seconds=timeout_seconds)
- manual_sign_in = None if update_ask else _manual_sign_in_offer(ctx, login_page_urls)
- if manual_sign_in is not None:
- # Signing in takes minutes in the same tab, so the card must survive a reload.
+ manual_sign_in = None if update_ask or registration else _manual_sign_in_offer(ctx, login_page_urls)
+ if manual_sign_in is not None or registration is not None:
+ # Signing in or saving a generated credential can outlast the tab, so the card must survive a reload.
ctx.credential_recovery_armed = True
recovery_enabled = _credential_recovery_enabled(ctx)
# Establish the active-pause record before the frame carries the token: the
@@ -953,6 +1067,7 @@ async def _run_credential_pause(
expires_at=expires_at,
anchor_tool_call_id=anchor_tool_call_id,
sign_in_browser_session_id=manual_sign_in.browser_session_id if manual_sign_in else None,
+ registration=registration,
timestamp=now,
)
await cache.set(
@@ -963,8 +1078,11 @@ async def _run_credential_pause(
card=card if recovery_enabled else None,
recovery_token_digest=ctx.credential_recovery_token_digest if recovery_enabled else None,
manual_sign_in=manual_sign_in,
+ registration=registration,
),
- ex=_longest_credential_pause_record_ttl() if manual_sign_in else _credential_pause_record_ttl(timeout_seconds),
+ ex=_longest_credential_pause_record_ttl()
+ if manual_sign_in or registration
+ else _credential_pause_record_ttl(timeout_seconds),
)
await stream.send(card)
@@ -991,9 +1109,16 @@ async def _run_credential_pause(
return "live"
await cache.set(
active_key,
- _encode_active_pause(resume_token, expires_at, consumed=True),
+ _encode_active_pause(
+ resume_token,
+ expires_at,
+ consumed=True,
+ registration=record.registration
+ if record is not None and record.resume_token == resume_token
+ else None,
+ ),
ex=_longest_credential_pause_record_ttl()
- if manual_sign_in
+ if manual_sign_in or registration
else _credential_pause_record_ttl(timeout_seconds),
)
return None
@@ -1060,6 +1185,23 @@ async def _run_credential_pause(
if resolution is None and not invalidated:
invalidated_resolution = await _invalidate_active_pause_record()
resolution = invalidated_resolution if isinstance(invalidated_resolution, CredentialPauseResolution) else None
+ generated_credential_id: str | None = None
+ if registration is not None:
+ settled_record = _decode_active_pause(
+ await cache.get(credential_pause_active_key(organization_id, chat_id, turn_id))
+ )
+ if settled_record is None or settled_record.resume_token != resume_token:
+ settled_record = None
+ settled_registration = settled_record.registration if settled_record is not None else None
+ generated_credential_id = settled_record.generated_credential_id if settled_record is not None else None
+ if settled_registration is not None:
+ ctx.credential_registration_outcome = settled_registration.outcome
+ # A create claimed but never finished (the pause ran out mid-create) may still have saved a credential.
+ connected = resolution is not None and resolution.action == "connected"
+ if settled_registration.attempted and settled_registration.outcome is None and not connected:
+ ctx.credential_registration_outcome = "unknown"
+ if generated_credential_id is not None or ctx.credential_registration_outcome == "unknown":
+ ctx.credential_generation_spent = True
if resolution is None:
settle("timeout")
return None
@@ -1095,7 +1237,9 @@ async def _run_credential_pause(
return None
await answered("connected", credential)
return resolution
- if admit_connected is not None and not await admit_connected(credential):
+ # A credential this card just generated has no saved site yet; the card's own origin is its only placement.
+ is_generated = credential.credential_id == generated_credential_id
+ if admit_connected is not None and not is_generated and not await admit_connected(credential):
ctx.credential_pause_outcome = "not_admitted"
await answered("not_admitted")
return resolution
@@ -1103,7 +1247,7 @@ async def _run_credential_pause(
_apply_connected_credential_to_policy(ctx, policy, credential)
ctx.credential_pause_outcome = "connected"
await answered("connected", credential)
- return resolution
+ return replace(resolution, generated=True) if is_generated else resolution
def arm_credential_pause_gate(ctx: Any) -> None:
@@ -1135,6 +1279,7 @@ async def request_credential_pause(
admit_connected: Callable[[Credential], Awaitable[bool]] | None = None,
allow_second_ask: bool = False,
anchor_tool_call_id: str | None = None,
+ registration: WorkflowCopilotCredentialRegistration | None = None,
) -> CredentialPauseResolution | None:
"""Raise the card from the model's own ``request_credential`` call and wait, inline, for the
answer, so tool calls the model issued alongside it can await ``credential_pause_settled``."""
@@ -1149,12 +1294,17 @@ async def request_credential_pause(
message,
stream,
copilot_config,
- reason=update_reason if update_ask else "login_credentials_unresolved",
+ reason=update_reason
+ if update_ask
+ else "credential_registration"
+ if registration
+ else "login_credentials_unresolved",
login_page_urls=[login_page_url],
update_credential_id=update_credential_id,
admit_connected=admit_connected,
allow_second_ask=allow_second_ask,
anchor_tool_call_id=anchor_tool_call_id,
+ registration=None if update_ask else registration,
)
if not update_ask:
ctx.credential_pause_reaskable_by_run = resolution is None or resolution.action == "skip"
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 1d53d9c8d..dc1c334aa 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -139,6 +139,11 @@ FINAL_REPLY_OBSERVATION = (
NUDGE_SENTINEL + "Your last response was empty. Tools are unavailable for this one response; "
"reply to the user now with your final response for this turn."
)
+RAW_SECRET_REPLY_WITHHELD_OBSERVATION = (
+ NUDGE_SENTINEL + "Your last reply was not shown to the user. The output check withheld it because it contained "
+ "text shaped like a raw secret: a value written after a password, passcode, secret, API key, or token label. "
+ "Tools are unavailable for this one response; reply to the user now with your final response for this turn."
+)
TOKEN_BUDGET = DEFAULT_TOKEN_BUDGET
SYNTHESIZED_BLOCK_PERSISTENCE_TOOL = "update_and_run_blocks"
# Both tools re-author the workflow draft and clear the coverage-reopen flag; the steer must fire
@@ -1418,6 +1423,7 @@ async def run_final_reply_drain(
session: Session | None,
hooks: FinalReplyRunHooks,
run_config: RunConfig,
+ observation: str = FINAL_REPLY_OBSERVATION,
) -> RunResultStreaming:
reply_run_config = replace(
run_config,
@@ -1429,7 +1435,7 @@ async def run_final_reply_drain(
try:
return await _run_streamed_with_deadline(
agent,
- FINAL_REPLY_OBSERVATION,
+ observation,
ctx,
session,
_SendTrackingStream(stream),
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 3be1aca51..60dcbd1cd 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -86,6 +86,7 @@ from skyvern.forge.sdk.copilot.workflow_yaml import (
stored_block_code,
stored_workflow_yaml,
)
+from skyvern.forge.sdk.schemas.workflow_copilot import CredentialRegistration
from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition
from skyvern.utils.yaml_loader import dump_workflow_yaml
@@ -219,12 +220,10 @@ from .mcp_hooks import get_skyvern_mcp_alias_map as get_skyvern_mcp_alias_map
from .page_challenge import FRESH_BROWSER_TOOL_NAME, SOLVE_TOOL_NAME, solve_page_challenge, start_fresh_browser
from .page_observation import _record_composition_page_observation as _record_composition_page_observation
from .page_observation import _resolve_url_title as _resolve_url_title
-from .run_execution import RUN_BLOCKS_STAGNATION_WINDOW_SECONDS as RUN_BLOCKS_STAGNATION_WINDOW_SECONDS
from .run_execution import (
RUN_RESULTS_MAX_ROW_KEYS,
)
from .run_execution import WatchdogExitReason as WatchdogExitReason
-from .run_execution import _any_quiet_block_requested as _any_quiet_block_requested
from .run_execution import _attach_action_traces as _attach_action_traces
from .run_execution import _block_end_urls_by_label as _block_end_urls_by_label
from .run_execution import _cancel_run_task_if_not_final as _cancel_run_task_if_not_final
@@ -238,7 +237,6 @@ from .run_execution import (
_diagnosis_repair_tool_error,
_get_run_results,
)
-from .run_execution import _progress_marker as _progress_marker
from .run_execution import _read_progress_sources as _read_progress_sources
from .run_execution import _record_diagnosis_repair_contract as _record_diagnosis_repair_contract
from .run_execution import _record_run_blocks_result as _record_run_blocks_result
@@ -995,6 +993,7 @@ async def request_credential_tool(
reason: str,
credential_id: str | None = None,
rejected_by_site: bool = False,
+ registration: CredentialRegistration | None = None,
) -> str:
"""Ask the user, in chat, how to sign in to a sign-in page: add or pick a saved credential, or
sign in themselves in the live browser and save that sign-in as a browser profile.
@@ -1011,6 +1010,14 @@ async def request_credential_tool(
refused that bound credential's saved password or code; a failed run's `credential_update` lever
names the one credential bound to its failed block. The card asks the user to update it and
comes back `updated` once they save; re-run the sign-in then.
+ When the user asked you to create one new account and the sign-up form needs a password, pass
+ `registration` with the sign-up page as `login_page_url`: the card offers to generate a password
+ and save it with that username and credential name, and you only ever get the credential's id.
+ `password_length` is 24 to 128; a longer or shorter site rule comes back `unsupported_constraints`.
+ `charset` is `alphanumeric` or `alphanumeric_symbols`; when the site's rules fit neither, call
+ without `registration` so the user adds the login. Never write a password yourself.
+ A second `registration` ask on a turn that already generated a credential, or may have, comes
+ back `already_generated` with no card.
The call waits for the user's answer and comes back `connected` with the credential
to bind, `skipped`, `unanswered`, or `unavailable` — follow the `next` or `fallback` it
carries. It comes back `signed_in` when the user signed in themselves in the live browser:
@@ -1023,6 +1030,7 @@ async def request_credential_tool(
"reason": reason,
"credential_id": credential_id,
"rejected_by_site": rejected_by_site,
+ "registration": registration.model_dump() if registration else None,
}
result: dict[str, Any] = {}
try:
@@ -1037,6 +1045,7 @@ async def request_credential_tool(
credential_id,
rejected_by_site,
anchor_tool_call_id=_originating_call_id(ctx),
+ registration=registration,
)
finally:
# A card on screen right now owns the gate; this call must not open it for one it never
@@ -1371,6 +1380,9 @@ async def get_run_results_tool(
returned run, excluding runs started by any Copilot chat, newest first; it
holds at most 5, so more may exist. When that lookup fails the result has
newer_finished_runs_unavailable instead, so a missing list is not "none".
+ execution_source with source_kind run_version describes the saved version the run executed, which may
+ differ from the current draft (current_draft_code_matches compares each code block); disposition
+ unavailable means that version cannot be proven, so no executed source is given.
"""
copilot_ctx = ctx.context
params: dict[str, Any] = {}
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index 9e3d280ef..fe9b1dca9 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -120,9 +120,8 @@ async def _call_internal_browser_tool(
_OUTCOME_EVIDENCE_BLOCK_TYPES = frozenset({BlockType.EXTRACTION.value, BlockType.VALIDATION.value})
-# Absolute upper bound on a single ``run_blocks`` tool invocation. Exists only
-# as a last-resort trip wire for runaway loops — progressing runs should never
-# approach this. The OpenAI Agents SDK wraps the tool in
+# Absolute upper bound on a single ``run_blocks`` tool invocation, including a run
+# whose rows have gone silent. The OpenAI Agents SDK wraps the tool in
# ``asyncio.wait_for(..., timeout=RUN_BLOCKS_SAFETY_CEILING_SECONDS)``; the
# inner poll loop leaves a 10 s headroom below this ceiling for orderly
# cleanup before the SDK cancels.
diff --git a/skyvern/forge/sdk/copilot/tools/credential_fill.py b/skyvern/forge/sdk/copilot/tools/credential_fill.py
index 5f0f1b571..72a0ecfc4 100644
--- a/skyvern/forge/sdk/copilot/tools/credential_fill.py
+++ b/skyvern/forge/sdk/copilot/tools/credential_fill.py
@@ -25,6 +25,10 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.credential_fill_fields import CREDENTIAL_FILL_FIELDS
+from skyvern.forge.sdk.copilot.credential_generation import (
+ REGISTRATION_PASSWORD_MAX_LENGTH,
+ REGISTRATION_PASSWORD_MIN_LENGTH,
+)
from skyvern.forge.sdk.copilot.credential_pause import (
RAW_SECRET_CONNECTED_NEXT,
CredentialPauseResolution,
@@ -76,6 +80,7 @@ from skyvern.forge.sdk.schemas.credentials import (
PasswordCredential,
TotpType,
)
+from skyvern.forge.sdk.schemas.workflow_copilot import CredentialRegistration, WorkflowCopilotCredentialRegistration
from skyvern.forge.sdk.services.credentials import generate_totp_code, normalize_totp_config
from skyvern.webeye.utils.dom import is_post_dispatch_click_timeout
@@ -490,6 +495,40 @@ _CREDENTIAL_CARD_FALLBACK = (
"Ask the user in prose to add the login on the Credentials page and reply with its exact saved name."
)
+_REGISTRATION_OUTCOME_NOTES: dict[str, dict[str, str]] = {
+ "rejected": {
+ "registration_outcome": "rejected",
+ "registration_detail": (
+ "Generate and save saved nothing: validation or the vault refused it, or the pre-check timed out."
+ ),
+ },
+ "unknown": {
+ "registration_outcome": "unknown",
+ "registration_detail": "Generate and save was not confirmed, so a credential under this name may exist.",
+ },
+}
+
+
+def _registration_ask_error(
+ registration: CredentialRegistration, policy: RequestPolicy, credential_id: str | None
+) -> dict[str, Any] | None:
+ if credential_id:
+ return {"ok": False, "error": "Pass either `registration` or `credential_id`, not both."}
+ if policy.raw_secret_detected:
+ return {"ok": False, "error": "Generating a credential is unavailable on a turn that contains a secret."}
+ if not registration.username.strip() or not registration.credential_name.strip():
+ return {"ok": False, "error": "`registration` needs a non-empty `username` and `credential_name`."}
+ if not REGISTRATION_PASSWORD_MIN_LENGTH <= registration.password_length <= REGISTRATION_PASSWORD_MAX_LENGTH:
+ return {
+ "ok": True,
+ "status": "unsupported_constraints",
+ "detail": (
+ f"Generated passwords are {REGISTRATION_PASSWORD_MIN_LENGTH} to "
+ f"{REGISTRATION_PASSWORD_MAX_LENGTH} characters, so no card was shown."
+ ),
+ }
+ return None
+
def _missing_totp_card_fallback(credential_name: str) -> str:
return (
@@ -590,6 +629,7 @@ async def _request_credential(
credential_id: str | None = None,
rejected_by_site: bool = False,
anchor_tool_call_id: str | None = None,
+ registration: CredentialRegistration | None = None,
) -> dict[str, Any]:
policy = copilot_ctx.request_policy
if not isinstance(policy, RequestPolicy) or (policy.raw_secret_detected and not policy.raw_secret_redacted_draft):
@@ -597,6 +637,29 @@ async def _request_credential(
ask_origin = canonicalize_origin(login_page_url)
if not is_resolved_page_url(login_page_url) or ask_origin is None:
return {"ok": False, "error": "Provide the absolute HTTP(S) sign-in page URL for the credential card."}
+ card_registration: WorkflowCopilotCredentialRegistration | None = None
+ if registration is not None:
+ registration_error = _registration_ask_error(registration, policy, credential_id)
+ if registration_error is not None:
+ return registration_error
+ if copilot_ctx.credential_generation_spent:
+ return {
+ "ok": True,
+ "status": "already_generated",
+ "detail": "Generate and save already ran on this turn, so no second generate card was shown.",
+ }
+ if not copilot_ctx.client_supports_credential_generation:
+ return {
+ "ok": True,
+ "status": "unavailable",
+ "detail": "Generating a credential from the card is not available on this turn.",
+ }
+ card_registration = WorkflowCopilotCredentialRegistration(
+ username=registration.username.strip(),
+ credential_name=registration.credential_name.strip(),
+ password_length=registration.password_length,
+ charset=registration.charset,
+ )
if policy.raw_secret_redacted_draft:
if credential_id:
return {
@@ -694,6 +757,7 @@ async def _request_credential(
admit_connected=admit_connected,
allow_second_ask=handback,
anchor_tool_call_id=anchor_tool_call_id,
+ registration=card_registration,
)
except BaseException:
if recovery is not None:
@@ -726,16 +790,21 @@ async def _request_credential(
clear_tool_blocker_signals_for_reason_codes(
copilot_ctx, frozenset({CREDENTIAL_ORIGIN_RECOVERY_PENDING_REASON_CODE})
)
+ registration_outcome = copilot_ctx.credential_registration_outcome if card_registration is not None else None
+ registration_note = _REGISTRATION_OUTCOME_NOTES[registration_outcome] if registration_outcome is not None else {}
if resolution is None:
- return {
+ unanswered: dict[str, Any] = {
"ok": True,
"status": "unanswered",
"outcome": copilot_ctx.credential_pause_outcome or "timeout",
- "next": (
+ **registration_note,
+ }
+ if card_registration is None:
+ unanswered["next"] = (
"The user did not answer the card. Continue without the credential: keep the credential "
"parameter placeholder in the draft and do not ask again this turn."
- ),
- }
+ )
+ return unanswered
if resolution.signed_in is not None:
return {
"ok": True,
@@ -756,26 +825,31 @@ async def _request_credential(
),
}
if credential is None:
- return {
- "ok": True,
- "status": "skipped",
- "next": (
+ skipped: dict[str, Any] = {"ok": True, "status": "skipped", **registration_note}
+ if card_registration is None:
+ skipped["next"] = (
"The user chose not to connect a credential now. Keep the credential parameter placeholder "
"in the draft, do not ask again this turn, and say a test run may stop at the login step."
- ),
- }
- return {
+ )
+ return skipped
+ connected: dict[str, Any] = {
"ok": True,
"status": "connected",
"credential_id": credential.credential_id,
"credential_name": credential.name,
- "next": (
- RAW_SECRET_CONNECTED_NEXT
- if policy.raw_secret_redacted_draft
- else "Bind this credential as the workflow's credential parameter and continue the build; run the "
- "blocks that were waiting on the login."
- ),
}
+ if policy.raw_secret_redacted_draft:
+ connected["next"] = RAW_SECRET_CONNECTED_NEXT
+ elif card_registration is not None:
+ if resolution.generated:
+ connected["generated"] = True
+ connected["site_effect"] = "none"
+ else:
+ connected["next"] = (
+ "Bind this credential as the workflow's credential parameter and continue the build; run the "
+ "blocks that were waiting on the login."
+ )
+ return connected
async def _missing_totp_ask_outcome(
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 8fa109ab5..509aad3c9 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -228,7 +228,11 @@ from skyvern.forge.sdk.copilot.turn_halt import (
stash_turn_halt_from_blocker_signal,
)
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
-from skyvern.forge.sdk.copilot.workflow_yaml import _process_workflow_yaml, runner_code_block_associations
+from skyvern.forge.sdk.copilot.workflow_yaml import (
+ _process_workflow_yaml,
+ runner_code_block_associations,
+ stored_block_code,
+)
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.db.enums import BrowserSeedSource
from skyvern.forge.sdk.executor.factory import AsyncExecutorFactory
@@ -268,6 +272,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (
WorkflowRunOutputParameter,
WorkflowRunParameter,
WorkflowRunStatus,
+ workflow_definition_sha256,
)
from skyvern.forge.sdk.workflow.runtime_completion import contract_from_request_criteria
from skyvern.forge.sdk.workflow.runtime_secret_bridge import consume_copilot_runtime_secret_values
@@ -356,19 +361,16 @@ _POST_RUN_PAGE_PARSE_TIMEOUT_SECONDS = 15.0
# starts with a ZIP local/central/spanning signature, so treat these prefixes as fail-closed.
_ZIP_MAGIC_PREFIXES = (b"PK\x03\x04", b"PK\x05\x06", b"PK\x07\x08")
-# Primary exit condition: seconds of no observed progress across the combined
-# run / block / step heartbeat. Sized to accommodate the slowest single LLM
-# round-trip (~30-60 s in practice) with headroom; going tighter risks
-# false-positives on healthy runs.
-RUN_BLOCKS_STAGNATION_WINDOW_SECONDS = 90
-
-
-# 5 s balances responsiveness (18 samples inside the stagnation window) against
-# DB load (240 polls worst case at the safety ceiling).
+# 5 s balances how quickly a terminal row is noticed against DB load (240 polls worst case at the safety ceiling).
RUN_BLOCKS_POLL_INTERVAL_SECONDS = 5.0
COPILOT_SANDBOX_UNAVAILABLE_ERROR = "Sandboxed worker is unavailable; execution was not started."
COPILOT_SNAPSHOT_PREPARATION_ERROR = "Unable to prepare the Copilot test-run snapshot; execution was not started."
+COPILOT_PROPOSAL_CHANGED_ERROR = "The pending Copilot proposal changed before the test started; reload and try again."
+COPILOT_PROPOSAL_SETTINGS_CHANGED_ERROR = (
+ "This request's workflow settings differ from the pending Copilot proposal's; "
+ "re-save the proposal with the current settings, then test again."
+)
# Block types that can reach exec() in the API process, so a run containing one may
# only proceed on the sandboxed worker. CODE compiles user code directly;
@@ -1264,128 +1266,42 @@ def _failure_action_trace_summary(failed_result: Mapping[str, Any] | None) -> li
return _retained_action_observations([failed_result])
-# Watchdog exit reasons. ``success`` means the run reached a trustworthy
-# terminal status inside the poll loop OR after the post-drain reconcile.
-# The run-ending reasons share the reconcile path but produce distinct
-# error messages: ``stagnation`` is the primary trip (no progress signals
-# for ``RUN_BLOCKS_STAGNATION_WINDOW_SECONDS`` seconds), ``ceiling`` is the
-# last-resort budget-exhausted branch, and ``task_exit_unfinalized`` is the
-# rare race where ``execute_workflow`` naturally exits before writing a
-# terminal row. ``paused`` is the exception: the run is alive and waiting on a
-# person, so it is neither cancelled nor reconciled.
+# ``success`` means a trustworthy terminal status was read in the poll loop or after the post-drain
+# reconcile; ``paused`` leaves a live run waiting on a person, so it is neither cancelled nor reconciled.
WatchdogExitReason = Literal[
"success",
- "stagnation",
"ceiling",
"task_exit_unfinalized",
"paused",
]
-# Block types that legitimately execute long silent periods: one DB write on
-# entry, work done without intermediate writes (sleep / LLM call / await human
-# input / browser download wait), one write on finish. The watchdog can't
-# distinguish these from "stuck", so any invocation that includes one disables
-# stagnation for the whole run and relies on the safety ceiling alone.
-_QUIET_BLOCK_TYPES: frozenset[str] = frozenset(
- {
- BlockType.WAIT.value,
- BlockType.TEXT_PROMPT.value,
- BlockType.WEB_SEARCH.value,
- BlockType.HUMAN_INTERACTION.value,
- BlockType.FILE_DOWNLOAD.value,
- BlockType.FILE_UPLOAD.value,
- # A code block writes its row on entry and exit and nothing in between, so a long
- # login or wait inside one is indistinguishable from a frozen run.
- BlockType.CODE.value,
- }
-)
-
-
-def _any_quiet_block_requested(
- copilot_ctx: CopilotContext,
- labels: list[str] | None,
- *,
- workflow: Workflow | None = None,
-) -> bool:
- """Return True if a selected block or its descendants can run without DB updates.
-
- Use the execution snapshot when supplied, including staged edits.
- """
- if not labels:
- return False
- last_workflow = workflow if workflow is not None else getattr(copilot_ctx, "last_workflow", None)
- if last_workflow is None:
- return False
- definition = getattr(last_workflow, "workflow_definition", None)
- pending = [(block, False) for block in (getattr(definition, "blocks", None) or [])]
- selected_labels = set(labels)
- while pending:
- block, parent_selected = pending.pop()
- selected = parent_selected or getattr(block, "label", None) in selected_labels
- pending.extend((child, selected) for child in _typed_child_blocks(block))
- if not selected:
- continue
- block_type = getattr(block, "block_type", None)
- if block_type is None:
- continue
- block_type_str = block_type.value if hasattr(block_type, "value") else str(block_type)
- if block_type_str in _QUIET_BLOCK_TYPES:
- return True
- return False
-
-
async def _read_progress_sources(
ctx: CopilotContext,
workflow_run_id: str,
-) -> tuple[WorkflowRun | None, datetime | None, datetime | None]:
- """Read one ``workflow_runs`` row + the two progress aggregates needed
- by the watchdog marker. Three cheap indexed queries; no row hydration
- on the aggregate side. The two repo calls run concurrently — they open
- separate async sessions and hit different tables.
- """
+) -> tuple[WorkflowRun | None, datetime | None]:
+ """Read one ``workflow_runs`` row and the latest block progress timestamp. The two repo calls run
+ concurrently because they open separate async sessions and hit different tables."""
- async def _read_timestamps() -> tuple[datetime | None, datetime | None]:
+ async def _read_block_timestamp() -> datetime | None:
try:
- return await app.DATABASE.tasks.get_workflow_run_progress_timestamps(
+ return await app.DATABASE.tasks.get_workflow_run_block_progress_timestamp(
workflow_run_id=workflow_run_id,
organization_id=ctx.organization_id,
)
except Exception:
LOG.warning(
- "Workflow run progress timestamps read failed",
+ "Workflow run block progress timestamp read failed",
workflow_run_id=workflow_run_id,
exc_info=True,
)
- return None, None
+ return None
- run, (step_ts, block_ts) = await asyncio.gather(
+ run, block_ts = await asyncio.gather(
_safe_read_workflow_run(workflow_run_id, ctx.organization_id, context="watchdog-poll"),
- _read_timestamps(),
- )
- return run, step_ts, block_ts
-
-
-def _progress_marker(
- run: WorkflowRun | None,
- step_ts: datetime | None,
- block_ts: datetime | None,
-) -> tuple[Any, ...]:
- """Hashable scalar snapshot. Changes iff any observable progress has
- occurred at the run, step, or block level since the last poll. Every
- ``update_step`` fires during action execution (including incremental
- token/cost accumulators at ``forge/agent.py:1449``), so
- ``max(step.modified_at)`` is the per-LLM-call heartbeat. Non-task blocks
- (CODE, TEXT_PROMPT) don't create step rows — ``max(workflow_run_block.modified_at)``
- covers that case. ``run.modified_at`` and ``run.status`` catch the
- run-level transitions that happen outside those two tables.
- """
- return (
- run.status if run else None,
- run.modified_at if run is not None else None,
- step_ts,
- block_ts,
+ _read_block_timestamp(),
)
+ return run, block_ts
async def _watchdog_error_message(
@@ -1406,15 +1322,7 @@ async def _watchdog_error_message(
ctx.origin_run_redaction_registry,
workflow_run_id,
)
- if exit_reason == "stagnation":
- body = (
- f"The run has not made progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s. "
- f"No step, block, or workflow-run row updates were observed in that window. "
- f"The page is most likely blocked by a captcha, popup, anti-bot challenge, "
- f"hidden validation error, or an infinite-retry loop on an action the agent "
- f"cannot detect is failing."
- )
- elif exit_reason == "paused":
+ if exit_reason == "paused":
# A pause is a healthy waiting state rather than an uncertain outcome, so it returns here
# instead of picking up the shared "outcome is uncertain, do not re-invoke" tail below. This
# is the one arm that directs the model to relay its own text, so it carries no run id.
@@ -1426,9 +1334,8 @@ async def _watchdog_error_message(
)
elif exit_reason == "ceiling":
body = (
- f"The run exceeded the {budget_seconds}s absolute ceiling "
- f"while still showing progress. The workflow is too long to fit in a single "
- f"tool invocation — split it into smaller block groups."
+ f"The run did not reach a terminal status within the {budget_seconds}s absolute ceiling. "
+ f"If the workflow is too long to fit in a single tool invocation, split it into smaller block groups."
)
else: # task_exit_unfinalized
last_observed = f"last observed status: {run.status}" if run is not None else "workflow run row was unreadable"
@@ -1453,12 +1360,10 @@ def _watchdog_user_facing_summary(
budget_seconds: int,
run: WorkflowRun | None,
) -> str:
- if exit_reason == "stagnation":
- return f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s."
if exit_reason == "paused":
return "The run is paused, waiting for a person to approve or reject it."
if exit_reason == "ceiling":
- return f"The run exceeded the {budget_seconds}s absolute ceiling while still showing progress."
+ return f"The run did not finish within the {budget_seconds}s absolute ceiling."
if run is not None:
return f"The run ended before recording a trustworthy terminal status. Last observed status: {run.status}."
return "The run ended before recording a trustworthy terminal status."
@@ -1794,6 +1699,182 @@ def _execution_source_facts(
}
+RunVersionUnavailableReason = Literal["version_missing", "identity_mismatch", "overwritten_after_run_start"]
+
+
+class _RunVersionSource(TypedDict):
+ """The saved version a finished run executed, read from the run's own workflow row."""
+
+ source_kind: Literal["run_version"]
+ disposition: Literal["available", "unavailable"]
+ workflow_run_id: str
+ workflow_id: str | None
+ reason: NotRequired[RunVersionUnavailableReason]
+ workflow_version: NotRequired[int]
+ failed_block_label: NotRequired[str]
+ failed_statement: NotRequired[str]
+ failed_statement_truncated: NotRequired[bool]
+ declared_parameter_keys: NotRequired[list[str]]
+ declared_parameter_keys_omitted: NotRequired[int]
+ block_code_sha256: NotRequired[dict[str, str]]
+ current_draft_code_matches: NotRequired[dict[str, bool]]
+ code_block_labels_omitted: NotRequired[int]
+ failed_block_code: NotRequired[str]
+ failed_block_code_truncated: NotRequired[bool]
+
+
+_RUN_VERSION_SOURCE_MAX_CHARS = 8000
+_RUN_VERSION_STATEMENT_MAX_CHARS = 600
+_RUN_VERSION_IDENTIFIER_MAX_CHARS = 120
+# Keys and labels stop this far short of the limit, so their omission counts and the code's truncation flag still fit.
+_RUN_VERSION_MARKER_RESERVE_CHARS = 256
+
+
+def _is_run_version_result(data: Mapping[str, Any]) -> bool:
+ source = data.get("execution_source")
+ return isinstance(source, Mapping) and source.get("source_kind") == "run_version"
+
+
+def _run_version_rewritten_after_start(run_workflow: Workflow, run_created_at: datetime) -> bool:
+ # A saved version can be overwritten in place after the run; a test version is only ever soft-deleted.
+ return run_workflow.created_by != COPILOT_TEST_WORKFLOW_CREATOR and _as_utc(run_workflow.modified_at) > _as_utc(
+ run_created_at
+ )
+
+
+def _run_version_unproven_reason(
+ ctx: CopilotContext, run: WorkflowRun, run_workflow: Workflow | None
+) -> RunVersionUnavailableReason | None:
+ if run_workflow is None:
+ return "version_missing"
+ if run_workflow.workflow_id != run.workflow_id or run_workflow.workflow_permanent_id != ctx.workflow_permanent_id:
+ return "identity_mismatch"
+ if run.workflow_definition_sha256 is not None:
+ current = workflow_definition_sha256(run_workflow.workflow_definition)
+ return None if current == run.workflow_definition_sha256 else "overwritten_after_run_start"
+ if _run_version_rewritten_after_start(run_workflow, run.created_at):
+ return "overwritten_after_run_start"
+ return None
+
+
+def _cut_to_serialized_chars(text: str, limit: int) -> str:
+ """Longest prefix whose JSON form fits ``limit``; escaping makes one character cost up to 12."""
+ low, high = 0, len(text)
+ while low < high:
+ middle = (low + high + 1) // 2
+ if len(json.dumps(text[:middle])) - 2 <= limit:
+ low = middle
+ else:
+ high = middle - 1
+ return text[:low]
+
+
+def _prompt_safe_text(ctx: CopilotContext, text: str) -> str:
+ """Exact-match scrubbing has to see the whole string, so this runs before any cut."""
+ return redact_raw_secrets_for_prompt(scrub_secrets_from_structure(ctx, text))
+
+
+def _run_version_source(
+ ctx: CopilotContext,
+ run: WorkflowRun,
+ run_workflow: Workflow | None,
+ *,
+ failed_block_label: str | None,
+ failing_line: int | None,
+ requested_block_labels: Sequence[str],
+) -> _RunVersionSource:
+ receipt: _RunVersionSource = {
+ "source_kind": "run_version",
+ "disposition": "unavailable",
+ "workflow_run_id": run.workflow_run_id,
+ "workflow_id": run.workflow_id,
+ }
+ reason = _run_version_unproven_reason(ctx, run, run_workflow)
+ if reason is not None or run_workflow is None:
+ receipt["reason"] = reason or "version_missing"
+ return receipt
+ receipt["disposition"] = "available"
+ receipt["workflow_version"] = run_workflow.version
+
+ code_inputs = _selected_code_security_inputs(
+ _workflow_definition_blocks_for_code_security(run_workflow.workflow_definition),
+ selected_labels=set(),
+ include_descendants=True,
+ )
+ safe_code_by_label = {code_input.label: _prompt_safe_text(ctx, code_input.code) for code_input in code_inputs}
+ raw_failed_code = next(
+ (code_input.code for code_input in code_inputs if code_input.label == failed_block_label), ""
+ )
+ safe_failed_code = safe_code_by_label.get(failed_block_label) if failed_block_label else None
+ if failed_block_label:
+ receipt["failed_block_label"] = _cut_to_serialized_chars(
+ _prompt_safe_text(ctx, failed_block_label), _RUN_VERSION_IDENTIFIER_MAX_CHARS
+ )
+ failed_lines = safe_failed_code.splitlines() if safe_failed_code else []
+ # Scrubbing a multi-line secret shifts every later line, so the recorded line number would name the wrong statement.
+ lines_kept_positions = len(failed_lines) == len(raw_failed_code.splitlines())
+ if failing_line is not None and lines_kept_positions and 0 < failing_line <= len(failed_lines):
+ statement = failed_lines[failing_line - 1].strip()
+ receipt["failed_statement"] = _cut_to_serialized_chars(statement, _RUN_VERSION_STATEMENT_MAX_CHARS)
+ if receipt["failed_statement"] != statement:
+ receipt["failed_statement_truncated"] = True
+
+ declared_keys = [
+ _cut_to_serialized_chars(_prompt_safe_text(ctx, parameter.key), _RUN_VERSION_IDENTIFIER_MAX_CHARS)
+ for parameter in _workflow_parameters(run_workflow)
+ if not isinstance(parameter, OutputParameter)
+ ]
+ receipt["declared_parameter_keys"] = []
+ for index, key in enumerate(declared_keys):
+ receipt["declared_parameter_keys"].append(key)
+ if len(json.dumps(receipt)) > _RUN_VERSION_SOURCE_MAX_CHARS - _RUN_VERSION_MARKER_RESERVE_CHARS:
+ receipt["declared_parameter_keys"].pop()
+ receipt["declared_parameter_keys_omitted"] = len(declared_keys) - index
+ break
+
+ draft_yaml = ctx.staged_workflow_yaml or ctx.workflow_yaml or ""
+ code_labels = [label for label in dict.fromkeys(requested_block_labels) if label in safe_code_by_label]
+ receipt["block_code_sha256"] = {}
+ receipt["current_draft_code_matches"] = {}
+ labels_omitted = 0
+ for index, label in enumerate(code_labels):
+ # Hashing scrubbed text keeps the digest from confirming a guessed secret next to the masked code.
+ digest = hashlib.sha256(safe_code_by_label[label].encode("utf-8")).hexdigest()
+ draft_code = stored_block_code(draft_yaml, label, allow_empty=True)
+ shown_label = _cut_to_serialized_chars(_prompt_safe_text(ctx, label), _RUN_VERSION_IDENTIFIER_MAX_CHARS)
+ if shown_label in receipt["block_code_sha256"]:
+ # Distinct labels can share a cut or scrubbed form; keep the first rather than merge their facts.
+ labels_omitted += 1
+ continue
+ receipt["block_code_sha256"][shown_label] = digest
+ receipt["current_draft_code_matches"][shown_label] = (
+ draft_code is not None
+ and hashlib.sha256(_prompt_safe_text(ctx, draft_code).encode("utf-8")).hexdigest() == digest
+ )
+ if len(json.dumps(receipt)) > _RUN_VERSION_SOURCE_MAX_CHARS - _RUN_VERSION_MARKER_RESERVE_CHARS:
+ del receipt["block_code_sha256"][shown_label]
+ del receipt["current_draft_code_matches"][shown_label]
+ labels_omitted += len(code_labels) - index
+ break
+ if labels_omitted:
+ receipt["code_block_labels_omitted"] = labels_omitted
+
+ if safe_failed_code:
+ receipt["failed_block_code"] = safe_failed_code
+ if len(json.dumps(receipt)) > _RUN_VERSION_SOURCE_MAX_CHARS:
+ receipt["failed_block_code_truncated"] = True
+ low, high = 0, len(safe_failed_code)
+ while low < high:
+ middle = (low + high + 1) // 2
+ receipt["failed_block_code"] = safe_failed_code[:middle]
+ if len(json.dumps(receipt)) <= _RUN_VERSION_SOURCE_MAX_CHARS:
+ low = middle
+ else:
+ high = middle - 1
+ receipt["failed_block_code"] = safe_failed_code[:low]
+ return receipt
+
+
_LOOP_INPUTS_MAX_LOOPS = 20
_LOOP_VALUES_COUNTS_MAX_ROWS = 10
@@ -1911,16 +1992,12 @@ def _attach_loop_inputs(
def _read_run_loop_inputs(
- run_workflow: Workflow | None, run_created_at: datetime, workflow_run_id: str, rows: Sequence[WorkflowRunBlock]
+ run_workflow: Workflow | None, workflow_run_id: str, rows: Sequence[WorkflowRunBlock]
) -> tuple[LoopInputs | LoopInputsUnavailable | None, list[str]]:
- loop_definitions = _loop_definitions_or_unavailable(run_workflow) if run_workflow is not None else None
- # A saved version can be overwritten in place after the run; a test version is only ever soft-deleted.
- if run_workflow is None or (
- run_workflow.created_by != COPILOT_TEST_WORKFLOW_CREATOR
- and _as_utc(run_workflow.modified_at) > _as_utc(run_created_at)
- ):
- ran_loop = loop_definitions is not None or any(row.block_type == BlockType.FOR_LOOP for row in rows)
+ if run_workflow is None:
+ ran_loop = any(row.block_type == BlockType.FOR_LOOP for row in rows)
return (LoopInputsUnavailable() if ran_loop else None), []
+ loop_definitions = _loop_definitions_or_unavailable(run_workflow)
if loop_definitions is None:
return None, []
return _observed_loop_inputs(loop_definitions, workflow_run_id, rows)
@@ -3699,6 +3776,14 @@ async def _run_blocks_and_collect_debug(
reused_origin_output_labels=reused_origin_output_labels,
origin_workflow_run_id=ctx.repair_origin_outputs_run_id if reused_origin_output_labels else None,
)
+ if ctx.settings_diverged_proposal_token == (execution.proposal_owner_turn_id, execution.proposal_revision):
+ LOG.info(
+ "copilot candidate run refused",
+ reason="settings_diverged",
+ proposal_owner_turn_id=execution.proposal_owner_turn_id,
+ proposal_revision=execution.proposal_revision,
+ )
+ return {"ok": False, "error": COPILOT_PROPOSAL_SETTINGS_CHANGED_ERROR}
if explicit_blank and snapshot.provenance == "canonical" and execution.source_is_current(ctx):
# Existing verification consumers need the same source this full test selected.
@@ -4127,187 +4212,191 @@ async def _run_blocks_and_collect_debug(
return _with_build_test_acquisition_context(session_err, requested_block_labels=block_labels)
run_session_id = ctx.browser_session_id
- execution.recorded_settings = await _browser_execution_settings(workflow, ctx, run_session_id)
-
- seeded_runtime_workflow = await _workflow_with_runtime_frontier_starter_url_seed(
- runtime_workflow,
- ctx,
- labels_to_execute=labels_to_execute,
- runtime_frontier_anchor_url=runtime_frontier_anchor_url,
- session_id_override=run_session_id,
- )
- runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow
- runtime_workflow = seeded_runtime_workflow
+ dispatch_draft_workflow_id: str | None = None
+ # run_task is the in-process inline execution task, only ever set on the dev-only inline path.
+ # For dispatched runs it stays None: the worker owns execution and the watchdog observes purely
+ # via DB polling.
+ run_task: asyncio.Task | None = None
+ sensitive_run_custody_lock: asyncio.Lock | None = None
+ sensitive_run_session_id: str | None = None
+ interim_run_id: str | None = None
+ # Set only where the run may already be executing, so an unwind before either executor call
+ # can still retract the run-start record.
+ run_submission_attempted = False
+ # Set before a re-entry of this call, which owns the minted browser from then on.
+ keep_run_session = False
+ try:
+ execution.recorded_settings = await _browser_execution_settings(workflow, ctx, run_session_id)
- requested_completion_contract = _requested_completion_contract(ctx, runtime_workflow, labels_to_execute)
- if requested_completion_contract is not None:
- runtime_workflow = runtime_workflow.model_copy(
- update={
- "workflow_definition": runtime_workflow.workflow_definition.model_copy(
- update={"completion_contract": requested_completion_contract}
- )
- }
+ seeded_runtime_workflow = await _workflow_with_runtime_frontier_starter_url_seed(
+ runtime_workflow,
+ ctx,
+ labels_to_execute=labels_to_execute,
+ runtime_frontier_anchor_url=runtime_frontier_anchor_url,
+ session_id_override=run_session_id,
)
+ runtime_frontier_starter_url_seeded = seeded_runtime_workflow is not runtime_workflow
+ runtime_workflow = seeded_runtime_workflow
- # Snapshot version persisted for a worker-dispatched run or an inline run of a definition that
- # was never persisted. The run is created against its exact workflow_id so prepare_workflow
- # reads parameter rows from the same definition execute_workflow receives. Without the inline
- # snapshot, newly drafted parameters exist only in memory and are omitted from the
- # WorkflowRunParameter rows, causing block execution to fail before it reaches the browser.
- # The snapshot is soft-deleted once the run resolves so it never lingers as the latest version.
- dispatch_draft_workflow_id: str | None = None
- # The persisted dispatch version (its own regenerated parameter ids) used for post-run output
- # mapping on the dispatch path; runtime_workflow / ctx.staged_workflow is left unmutated.
- dispatch_workflow: Workflow | None = None
- if dispatch_to_worker or snapshot.provenance == "staged" or runtime_workflow != snapshot.workflow:
- # Persist the wrapped runtime workflow as a real new version (with its own parameter /
- # output-parameter rows) through the normal create machinery. The run is then created
- # against this version so the worker resolves it by run.workflow_id and registers block
- # outputs from the version's own rows.
- try:
- dispatch_workflow = await app.WORKFLOW_SERVICE.create_copilot_dispatch_draft_version(
- runtime_workflow=runtime_workflow,
- organization_id=ctx.organization_id,
- )
- dispatch_draft_workflow_id = dispatch_workflow.workflow_id
- except Exception:
- LOG.warning(
- "Failed to persist copilot run snapshot; blocking execution",
- organization_id=ctx.organization_id,
- workflow_permanent_id=ctx.workflow_permanent_id,
- dispatch_to_worker=dispatch_to_worker,
- snapshot_provenance=snapshot.provenance,
- exc_info=True,
+ requested_completion_contract = _requested_completion_contract(ctx, runtime_workflow, labels_to_execute)
+ if requested_completion_contract is not None:
+ runtime_workflow = runtime_workflow.model_copy(
+ update={
+ "workflow_definition": runtime_workflow.workflow_definition.model_copy(
+ update={"completion_contract": requested_completion_contract}
+ )
+ }
)
- return {
- "ok": False,
- "error": COPILOT_SNAPSHOT_PREPARATION_ERROR,
- "data": {
- "workflow_run_id": None,
- "failure_reason": COPILOT_SNAPSHOT_PREPARATION_ERROR,
- "blocks": [],
- },
- }
- if dispatch_workflow is not None:
- snapshot = _materialized_execution_snapshot(snapshot, dispatch_workflow)
- # Read before execution: a natural-language loop reference is rewritten on the block while it runs.
- execution.loop_definitions = _loop_definitions_or_unavailable(snapshot.workflow)
-
- if execution.recorded_settings is not None:
- # Persistence owns model/header/profile-key settings; the attached browser owns
- # the proxy and loaded profile already observed above. None stays unproven.
- execution.recorded_settings = copy.deepcopy(
- replace(
- OriginExecutionSettings.of(snapshot.workflow),
- proxy_location=execution.recorded_settings.proxy_location,
- browser_profile_id=execution.recorded_settings.browser_profile_id,
+ # Snapshot version persisted for a worker-dispatched run or an inline run of a definition that
+ # was never persisted. The run is created against its exact workflow_id so prepare_workflow
+ # reads parameter rows from the same definition execute_workflow receives. Without the inline
+ # snapshot, newly drafted parameters exist only in memory and are omitted from the
+ # WorkflowRunParameter rows, causing block execution to fail before it reaches the browser.
+ # The snapshot is soft-deleted once the run resolves so it never lingers as the latest version.
+ # The persisted dispatch version (its own regenerated parameter ids) used for post-run output
+ # mapping on the dispatch path; runtime_workflow / ctx.staged_workflow is left unmutated.
+ dispatch_workflow: Workflow | None = None
+ if dispatch_to_worker or snapshot.provenance == "staged" or runtime_workflow != snapshot.workflow:
+ # Persist the wrapped runtime workflow as a real new version (with its own parameter /
+ # output-parameter rows) through the normal create machinery. The run is then created
+ # against this version so the worker resolves it by run.workflow_id and registers block
+ # outputs from the version's own rows.
+ try:
+ dispatch_workflow = await app.WORKFLOW_SERVICE.create_copilot_dispatch_draft_version(
+ runtime_workflow=runtime_workflow,
+ organization_id=ctx.organization_id,
+ )
+ dispatch_draft_workflow_id = dispatch_workflow.workflow_id
+ except Exception:
+ LOG.warning(
+ "Failed to persist copilot run snapshot; blocking execution",
+ organization_id=ctx.organization_id,
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ dispatch_to_worker=dispatch_to_worker,
+ snapshot_provenance=snapshot.provenance,
+ exc_info=True,
+ )
+ return {
+ "ok": False,
+ "error": COPILOT_SNAPSHOT_PREPARATION_ERROR,
+ "data": {
+ "workflow_run_id": None,
+ "failure_reason": COPILOT_SNAPSHOT_PREPARATION_ERROR,
+ "blocks": [],
+ },
+ }
+
+ if dispatch_workflow is not None:
+ snapshot = _materialized_execution_snapshot(snapshot, dispatch_workflow)
+ # Read before execution: a natural-language loop reference is rewritten on the block while it runs.
+ execution.loop_definitions = _loop_definitions_or_unavailable(snapshot.workflow)
+
+ if execution.recorded_settings is not None:
+ # Persistence owns model/header/profile-key settings; the attached browser owns
+ # the proxy and loaded profile already observed above. None stays unproven.
+ execution.recorded_settings = copy.deepcopy(
+ replace(
+ OriginExecutionSettings.of(snapshot.workflow),
+ proxy_location=execution.recorded_settings.proxy_location,
+ browser_profile_id=execution.recorded_settings.browser_profile_id,
+ )
)
- )
- execution.dispatched_output_parameter_ids = {
- block.label: block.output_parameter.output_parameter_id
- for block in snapshot.workflow.workflow_definition.blocks
- }
+ execution.dispatched_output_parameter_ids = {
+ block.label: block.output_parameter.output_parameter_id
+ for block in snapshot.workflow.workflow_definition.blocks
+ }
- all_workflow_params = list(snapshot.workflow_parameters)
- all_output_params = list(snapshot.output_parameters)
- # The check above resolved every key the dispatched run reads only if persistence added none and dropped none.
- unchecked_parameter_keys = (
- sorted(origin_checked_parameter_keys ^ {parameter.key for parameter in all_workflow_params})
- if origin_checked_resolution is not None
- else []
- )
- refusal = (
- selected_output_run_refusal(
- ctx,
- execution.selected_output_sources,
- workflow,
- data,
- labels_to_execute[0],
- execution_settings=execution.recorded_settings,
+ all_workflow_params = list(snapshot.workflow_parameters)
+ all_output_params = list(snapshot.output_parameters)
+ # The check above resolved every key the dispatched run reads only if persistence added none and dropped none.
+ unchecked_parameter_keys = (
+ sorted(origin_checked_parameter_keys ^ {parameter.key for parameter in all_workflow_params})
+ if origin_checked_resolution is not None
+ else []
)
- if execution.selected_output_sources
- else None
- )
- if unchecked_parameter_keys and (execution.selected_output_sources or execution.reused_origin_output_labels):
- refusal = logged_origin_refusal(
- OriginOutputRefusalDetail(
- reason=OriginOutputRefusal.CHANGED_INPUT,
- block_label=next(iter(execution.selected_output_sources), None)
- or execution.reused_origin_output_labels[0],
- origin_workflow_run_id=(
- next(iter(execution.selected_output_sources.values())).workflow_run_id
- if execution.selected_output_sources
- else execution.origin_workflow_run_id
- ),
- parameter_key=unchecked_parameter_keys[0],
+ refusal = (
+ selected_output_run_refusal(
+ ctx,
+ execution.selected_output_sources,
+ workflow,
+ data,
+ labels_to_execute[0],
+ execution_settings=execution.recorded_settings,
)
+ if execution.selected_output_sources
+ else None
)
- if refusal is not None:
- # Persistence follows the browser-derived starter URL. Nothing has executed yet:
- # retire that preparation and send the original request through security, input
- # resolution and browser selection again. With no selected sources this refusal
- # cannot trigger another re-entry, even if persistence changes parameter keys again.
- if dispatch_draft_workflow_id is not None:
- await _delete_dispatch_draft(dispatch_draft_workflow_id, ctx.organization_id)
- # Nothing has run in a verified new-exit browser yet, so the re-entry reuses it instead of paying for another.
- if used_fresh_run_session and run_session_id and new_exit_receipt is None:
- await close_browser_session_quietly(ctx.organization_id, run_session_id)
- ctx.frontier_selected_output_sources = {}
- ctx.frontier_origin_reused_labels = []
- ctx.frontier_resume_session_id = None
- ctx.frontier_requires_own_browser = False
- requested_labels, requested_seed, requested_start, requested_provenance = _anchored_plan(
- ctx, (list(block_labels), {}, block_labels[0]), workflow.workflow_definition, None, list(block_labels)
- )
- ctx.frontier_start_provenance = requested_provenance
- ctx.frontier_origin_output_refusal = refusal
- return await _run_blocks_and_collect_debug(
- params,
- ctx,
- labels_to_execute=requested_labels,
- block_outputs_to_seed=requested_seed,
- frontier_start_label=requested_start,
- force_fresh_session=force_fresh_session,
- execution_snapshot=execution.snapshot,
- explicit_blank=explicit_blank,
- use_ephemeral_inputs=use_ephemeral_inputs,
- new_exit=new_exit,
- verified_new_exit=new_exit_receipt,
- )
-
- execution.dispatched_input_values = copy.deepcopy(data)
- ctx.unbound_required_parameter_keys = unbound_required_parameter_keys
- execution.reused_origin_input_keys = reused_origin_input_keys
- execution.unbound_keys = list(ctx.unbound_required_parameter_keys)
- # Only credential-typed values are ever read back; scout-typed form inputs stay out of the record.
- execution.parameter_values = {
- parameter.key: data[parameter.key]
- for parameter in all_workflow_params
- if parameter.key in data and parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID
- }
+ if unchecked_parameter_keys and (execution.selected_output_sources or execution.reused_origin_output_labels):
+ refusal = logged_origin_refusal(
+ OriginOutputRefusalDetail(
+ reason=OriginOutputRefusal.CHANGED_INPUT,
+ block_label=next(iter(execution.selected_output_sources), None)
+ or execution.reused_origin_output_labels[0],
+ origin_workflow_run_id=(
+ next(iter(execution.selected_output_sources.values())).workflow_run_id
+ if execution.selected_output_sources
+ else execution.origin_workflow_run_id
+ ),
+ parameter_key=unchecked_parameter_keys[0],
+ )
+ )
+ if refusal is not None:
+ # Persistence follows the browser-derived starter URL. Nothing has executed yet:
+ # retire that preparation and send the original request through security, input
+ # resolution and browser selection again. With no selected sources this refusal
+ # cannot trigger another re-entry, even if persistence changes parameter keys again.
+ if dispatch_draft_workflow_id is not None:
+ await _delete_dispatch_draft(dispatch_draft_workflow_id, ctx.organization_id)
+ dispatch_draft_workflow_id = None
+ # Nothing has run in a verified new-exit browser yet, so the re-entry reuses it instead of paying for another.
+ if used_fresh_run_session and run_session_id and new_exit_receipt is None:
+ await close_browser_session_quietly(ctx.organization_id, run_session_id)
+ keep_run_session = True
+ ctx.frontier_selected_output_sources = {}
+ ctx.frontier_origin_reused_labels = []
+ ctx.frontier_resume_session_id = None
+ ctx.frontier_requires_own_browser = False
+ requested_labels, requested_seed, requested_start, requested_provenance = _anchored_plan(
+ ctx, (list(block_labels), {}, block_labels[0]), workflow.workflow_definition, None, list(block_labels)
+ )
+ ctx.frontier_start_provenance = requested_provenance
+ ctx.frontier_origin_output_refusal = refusal
+ return await _run_blocks_and_collect_debug(
+ params,
+ ctx,
+ labels_to_execute=requested_labels,
+ block_outputs_to_seed=requested_seed,
+ frontier_start_label=requested_start,
+ force_fresh_session=force_fresh_session,
+ execution_snapshot=execution.snapshot,
+ explicit_blank=explicit_blank,
+ use_ephemeral_inputs=use_ephemeral_inputs,
+ new_exit=new_exit,
+ verified_new_exit=new_exit_receipt,
+ )
- workflow_request = WorkflowRequestBody(
- data=data if data else None,
- browser_session_id=run_session_id,
- # Copilot test runs don't need scrolling post-action screenshots;
- # the ForgeAgent's split screenshots (used for LLM context) are unaffected.
- max_screenshot_scrolls=0,
- )
+ execution.dispatched_input_values = copy.deepcopy(data)
+ ctx.unbound_required_parameter_keys = unbound_required_parameter_keys
+ execution.reused_origin_input_keys = reused_origin_input_keys
+ execution.unbound_keys = list(ctx.unbound_required_parameter_keys)
+ # Only credential-typed values are ever read back; scout-typed form inputs stay out of the record.
+ execution.parameter_values = {
+ parameter.key: data[parameter.key]
+ for parameter in all_workflow_params
+ if parameter.key in data and parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID
+ }
- # run_task is the in-process inline execution task, only ever set on the dev-only inline path.
- # For dispatched runs it stays None: the worker owns execution and the watchdog observes purely
- # via DB polling.
- run_task: asyncio.Task | None = None
- sensitive_run_custody_lock: asyncio.Lock | None = None
- sensitive_run_session_id: str | None = None
- interim_run_id: str | None = None
- current_context = skyvern_context.current()
- # Set only where the run may already be executing, so an unwind before either executor call
- # can still retract the run-start record.
- run_submission_attempted = False
- try:
+ workflow_request = WorkflowRequestBody(
+ data=data if data else None,
+ browser_session_id=run_session_id,
+ # Copilot test runs don't need scrolling post-action screenshots;
+ # the ForgeAgent's split screenshots (used for LLM context) are unaffected.
+ max_screenshot_scrolls=0,
+ )
+
+ current_context = skyvern_context.current()
workflow_run = await workflow_service.prepare_workflow(
workflow_id=ctx.workflow_permanent_id,
organization=organization,
@@ -4355,8 +4444,6 @@ async def _run_blocks_and_collect_debug(
if dispatch_draft_workflow_id is not None:
await _delete_dispatch_draft(dispatch_draft_workflow_id, ctx.organization_id)
dispatch_draft_workflow_id = None
- if run_session_id:
- await close_browser_session_quietly(ctx.organization_id, run_session_id)
return {
"ok": False,
"error": "The test run's saved browser profile could not be recorded; execution was not started.",
@@ -4402,10 +4489,7 @@ async def _run_blocks_and_collect_debug(
proposal_owner_turn_id=execution.proposal_owner_turn_id,
proposal_revision=execution.proposal_revision,
)
- return {
- "ok": False,
- "error": "The pending Copilot proposal changed before the test started; reload and try again.",
- }
+ return {"ok": False, "error": COPILOT_PROPOSAL_CHANGED_ERROR}
ctx.dispatched_run_ids_this_turn.add(workflow_run.workflow_run_id)
# A seeded producer's value belongs to the run that produced it, never to this one.
seeded_only_labels = frozenset(block_outputs_to_seed) - frozenset(labels_that_may_execute)
@@ -4541,6 +4625,11 @@ async def _run_blocks_and_collect_debug(
if interim_run_id is not None and not run_submission_attempted:
_forget_interim_run_start(ctx, interim_run_id)
raise
+ finally:
+ # Every exit before submission, cancellation included, releases the minted browser; once submission was
+ # attempted the run may already hold it.
+ if used_fresh_run_session and run_session_id and not run_submission_attempted and not keep_run_session:
+ await close_browser_session_quietly(ctx.organization_id, run_session_id)
active_run_association: ActiveRunSessionAssociation | None = None
run_paused = False
@@ -4575,18 +4664,12 @@ async def _run_blocks_and_collect_debug(
# completion after the SSE stream is gone so its reply persists
# (SKY-8986); aborting mid-block would strand the run without debug
# output for the final chat message.
- initial_run, initial_step_ts, initial_block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)
- progress_marker = _progress_marker(initial_run, initial_step_ts, initial_block_ts)
- last_progress_monotonic = time.monotonic()
- started_monotonic = last_progress_monotonic
+ initial_run, initial_block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)
+ started_monotonic = time.monotonic()
final_status: str | None = None
run: Any = initial_run
exit_reason: WatchdogExitReason | None = None
run_cancelled_by_watchdog = False
- # Quiet blocks (WAIT/TEXT_PROMPT/HUMAN_INTERACTION) legitimately have
- # DB-silent periods; disable stagnation for any invocation that includes
- # one. Safety ceiling still applies.
- stagnation_enabled = not _any_quiet_block_requested(ctx, labels_that_may_execute, workflow=runtime_workflow)
budget_seconds = max(1, RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10)
narrator_state: NarratorState | None = ctx.narrator_state
@@ -4598,7 +4681,7 @@ async def _run_blocks_and_collect_debug(
while True:
await asyncio.sleep(RUN_BLOCKS_POLL_INTERVAL_SECONDS)
- run, step_ts, block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)
+ run, block_ts = await _read_progress_sources(ctx, workflow_run.workflow_run_id)
if progress_enabled:
assert narrator_state is not None
@@ -4632,26 +4715,12 @@ async def _run_blocks_and_collect_debug(
exit_reason = "task_exit_unfinalized"
break
- now = time.monotonic()
- new_marker = _progress_marker(run, step_ts, block_ts)
- # A run in ``paused`` status (e.g. HumanInteractionBlock) is a
- # user-driven wait, not stagnation — never trip.
- is_paused = run is not None and run.status == WorkflowRunStatus.paused.value
- stagnation_active = stagnation_enabled and not is_paused
-
- if new_marker != progress_marker:
- progress_marker = new_marker
- last_progress_monotonic = now
- elif stagnation_active and now - last_progress_monotonic >= RUN_BLOCKS_STAGNATION_WINDOW_SECONDS:
- exit_reason = "stagnation"
- break
-
- if is_paused:
+ if run is not None and run.status == WorkflowRunStatus.paused.value:
exit_reason = "paused"
run_paused = True
break
- if now - started_monotonic >= budget_seconds:
+ if time.monotonic() - started_monotonic >= budget_seconds:
exit_reason = "ceiling"
break
@@ -4664,7 +4733,7 @@ async def _run_blocks_and_collect_debug(
# synthetic-``canceled`` ambiguity that the post-drain reread
# has to exclude. Then cancel + reread +
# ``_trusted_post_drain_status`` applies SKY-9167's success-race
- # recovery uniformly to all three non-success exit reasons.
+ # recovery uniformly to both non-success exit reasons.
pre_cancel_run = await _safe_read_workflow_run(
workflow_run.workflow_run_id, ctx.organization_id, context="pre-cancel"
)
@@ -5230,6 +5299,14 @@ async def _get_run_results(
run_workflow = None
if run_workflow is None:
LOG.warning("Prior-run workflow snapshot fetch failed", workflow_run_id=workflow_run_id)
+ # Only a row proven to be what the run executed may describe it; an unproven row is treated as missing,
+ # which every consumer below already handles fail-closed (including sensitivity).
+ executed_workflow = (
+ run_workflow
+ if workflow_run_id == ctx.proposal_workflow_run_id
+ or _run_version_unproven_reason(ctx, run, run_workflow) is None
+ else None
+ )
origin_registry = getattr(ctx, "origin_run_redaction_registry", None)
matching_origin_registry = (
@@ -5237,7 +5314,7 @@ async def _get_run_results(
)
# Unknown provenance is not evidence that a cold run was nonsensitive. Use the same
# fail-closed classification for every direct page producer and for terminal artifacts.
- workflow_has_sensitive_parameters = _workflow_requires_terminal_artifact_redaction(run_workflow)
+ workflow_has_sensitive_parameters = _workflow_requires_terminal_artifact_redaction(executed_workflow)
sensitive_origin_run = (
_origin_registry_contains_sensitive_values(matching_origin_registry, workflow_run_id)
or workflow_has_sensitive_parameters
@@ -5287,9 +5364,19 @@ async def _get_run_results(
"action_trace_summary": action_trace_summary,
"action_observations": action_observations,
}
- if run_workflow is not None and workflow_run_id == ctx.proposal_workflow_run_id:
- result_data["execution_source"] = _execution_source_facts(run_workflow, provenance="staged")
- loop_inputs, loop_input_notices = _read_run_loop_inputs(run_workflow, run.created_at, workflow_run_id, blocks)
+ if workflow_run_id == ctx.proposal_workflow_run_id:
+ if run_workflow is not None:
+ result_data["execution_source"] = _execution_source_facts(run_workflow, provenance="staged")
+ else:
+ result_data["execution_source"] = _run_version_source(
+ ctx,
+ run,
+ run_workflow,
+ failed_block_label=_packet_string(newest_failed.get("label")) if newest_failed else None,
+ failing_line=result_data["failing_code_line"],
+ requested_block_labels=result_data["requested_block_labels"],
+ )
+ loop_inputs, loop_input_notices = _read_run_loop_inputs(executed_workflow, workflow_run_id, blocks)
if loop_inputs is not None:
result_data[LOOP_INPUTS_KEY] = loop_inputs.model_dump(mode="json", exclude_none=True)
if loop_input_notices:
@@ -5314,7 +5401,9 @@ async def _get_run_results(
and not sensitive_origin_run
and not _run_browser_carries_a_sign_in(ctx, run.browser_session_id)
):
- failed_block_code = _failed_block_code(run_workflow, newest_failed) if run_workflow is not None else None
+ failed_block_code = (
+ _failed_block_code(executed_workflow, newest_failed) if executed_workflow is not None else None
+ )
locator_observations = await _observe_authored_locators(
ctx,
run_session_id=run.browser_session_id,
@@ -5351,26 +5440,30 @@ async def _get_run_results(
workflow_run_id=workflow_run_id,
)
- if run_workflow is None:
+ if executed_workflow is None:
result_data["requested_output_definitions_omission"] = "the run-pinned workflow snapshot was unavailable"
else:
result_data["requested_output_parameter_definitions"] = _requested_output_parameter_definitions(
workflow_run_id=workflow_run_id,
- workflow=run_workflow,
+ workflow=executed_workflow,
)
await _attach_registered_output_parameter_values(
workflow_run_id=workflow_run_id,
- workflow=run_workflow,
+ workflow=executed_workflow,
data=result_data,
persisted_output_parameters=(
- [parameter for parameter in _workflow_parameters(run_workflow) if isinstance(parameter, OutputParameter)]
- if run_workflow is not None
+ [
+ parameter
+ for parameter in _workflow_parameters(executed_workflow)
+ if isinstance(parameter, OutputParameter)
+ ]
+ if executed_workflow is not None
else None
),
excluded_block_labels=ctx.seeded_only_labels_by_run_id.get(workflow_run_id, frozenset()),
)
- cold_artifact_requires_redaction_context = _workflow_requires_terminal_artifact_redaction(run_workflow)
+ cold_artifact_requires_redaction_context = _workflow_requires_terminal_artifact_redaction(executed_workflow)
# Cold ordinary-turn hydration did not perform a sensitive run and cannot use mutable context
# alone to vouch for its raw artifact custody. The direct same-turn tool route may use its exact
# immutable registry; cold sensitive hydration receives a typed omission instead.
@@ -5389,7 +5482,7 @@ async def _get_run_results(
run_id=workflow_run_id,
organization_id=ctx.organization_id,
current_url=current_url,
- workflow=run_workflow,
+ workflow=executed_workflow,
origin_redaction_registry=artifact_redaction_registry,
)
)
@@ -6411,7 +6504,14 @@ def _build_recorded_build_test_outcome(
workflow_definition = getattr(copilot_ctx.last_workflow, "workflow_definition", None)
raw_requested_block_labels = getattr(copilot_ctx, "last_requested_block_labels", None)
execution = result.execution if isinstance(result, _ExecutionResult) else None
- if execution is not None:
+ historical_run = _is_run_version_result(result_data)
+ if historical_run:
+ # The chat's draft is not what a finished run executed, so none of its signatures describe this run.
+ workflow_yaml = None
+ code_artifact_metadata = {}
+ workflow_definition = None
+ raw_requested_block_labels = result_data.get("requested_block_labels")
+ elif execution is not None:
workflow_yaml = execution.workflow_yaml
code_artifact_metadata = execution.metadata
workflow_definition = execution.snapshot.workflow.workflow_definition
@@ -6449,7 +6549,7 @@ def _build_recorded_build_test_outcome(
declared_goal_path_omissions=declared_goal_path_omissions,
unbound_required_parameter_keys=execution.unbound_keys
if execution
- else list(copilot_ctx.unbound_required_parameter_keys),
+ else ([] if historical_run else list(copilot_ctx.unbound_required_parameter_keys)),
block_parameter_keys=authored_block_parameter_keys_from_workflow(
workflow_yaml,
code_artifact_metadata,
@@ -6457,7 +6557,7 @@ def _build_recorded_build_test_outcome(
block_shape_hashes=block_shape_hashes,
block_associations_by_label=execution.associations
if execution
- else getattr(copilot_ctx, "runner_code_block_associations_by_label", {}),
+ else ({} if historical_run else getattr(copilot_ctx, "runner_code_block_associations_by_label", {})),
)
raw_blocks = result_data.get("blocks")
ai_fallback_blocks = _packet_ai_fallback_blocks(raw_blocks if isinstance(raw_blocks, list) else [])
@@ -7249,20 +7349,21 @@ def _packet_unfinished_items(
recorded_outcome: RecordedBuildTestOutcome | None,
declared_goal_path_omissions: object,
omission_notices: list[str],
+ unverified_block_labels: Sequence[str],
) -> list[BuildTestPacketUnfinishedItem]:
latest_outcome = getattr(copilot_ctx, "latest_recorded_build_test_outcome", None)
outcome = recorded_outcome or (latest_outcome if isinstance(latest_outcome, RecordedBuildTestOutcome) else None)
if run_id is None:
- if copilot_ctx.last_unverified_block_labels or outcome is not None:
+ if unverified_block_labels or outcome is not None:
omission_notices.append("unfinished_items omitted: recorded unfinished evidence is not bound to this run.")
return []
outcome_matches_run = outcome is not None and outcome.workflow_run_id == run_id
- if not outcome_matches_run and (copilot_ctx.last_unverified_block_labels or outcome is not None):
+ if not outcome_matches_run and (unverified_block_labels or outcome is not None):
omission_notices.append("unfinished_items omitted: recorded unfinished evidence is not bound to this run.")
unfinished: list[BuildTestPacketUnfinishedItem] = (
[
BuildTestPacketUnfinishedItem(kind="unverified_block", label=label)
- for label in dict.fromkeys(copilot_ctx.last_unverified_block_labels)
+ for label in dict.fromkeys(unverified_block_labels)
if isinstance(label, str) and label
]
if outcome_matches_run
@@ -7472,6 +7573,8 @@ def build_test_evidence_packet(
recorded_outcome,
raw_goal_path_omissions,
omission_notices,
+ # The chat's unverified labels describe its current draft, not the version a finished run executed.
+ unverified_block_labels=[] if _is_run_version_result(data) else copilot_ctx.last_unverified_block_labels,
)
if not unfinished_items:
omission_notices.append("unfinished_items empty: recorded outcome and workflow evidence identify none.")
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 25245cd05..66b4cddf7 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -16,7 +16,7 @@ from typing import Annotated, Any, Literal, NamedTuple, cast
import structlog
import yaml
from jinja2 import TemplateSyntaxError
-from pydantic import AliasChoices, BaseModel, Field, ValidationError
+from pydantic import AliasChoices, BaseModel, Field, TypeAdapter, ValidationError
from skyvern.constants import DEFAULT_WORKFLOW_TITLES
from skyvern.exceptions import SkyvernHTTPException
@@ -114,6 +114,8 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (
workflow_blocks,
)
from skyvern.forge.sdk.copilot.workflow_yaml import (
+ _PRIVATE_WORKFLOW_SETTINGS_FIELDS,
+ _canonicalize_copilot_proxy_location,
_normalize_copilot_yaml,
_process_workflow_yaml,
dump_workflow_yaml,
@@ -140,7 +142,7 @@ from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, Ins
from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.workflow import Workflow
from skyvern.forge.sdk.workflow.runtime_completion import contract_from_code_artifact_metadata
-from skyvern.schemas.proxy_location import runtime_proxy_location
+from skyvern.schemas.proxy_location import ProxyLocationInput, runtime_proxy_location
from skyvern.schemas.workflows import (
BLOCK_YAML_SUBCLASSES,
BlockType,
@@ -3990,7 +3992,9 @@ def strip_copilot_yaml_headers(workflow_yaml: str | None) -> str | None:
return None
-def copilot_workflow_yaml_matches(submitted_yaml: str | None, *known_sources: str | None) -> bool:
+def copilot_workflow_yaml_matches(
+ submitted_yaml: str | None, *known_sources: str | None, inherited_proxy_location: ProxyLocationInput = None
+) -> bool:
submitted_yaml = strip_copilot_yaml_headers(submitted_yaml)
if not submitted_yaml:
return False
@@ -3999,10 +4003,14 @@ def copilot_workflow_yaml_matches(submitted_yaml: str | None, *known_sources: st
sanitized_source = strip_copilot_yaml_headers(source)
if not sanitized_source:
continue
- source_canvas = _normalized_canvas_for_proposal_restore(sanitized_source)
+ source_canvas = _normalized_canvas_for_proposal_restore(
+ sanitized_source, inherited_proxy_location=inherited_proxy_location
+ )
if (
_normalized_canvas_for_proposal_restore(
- submitted_yaml, inherited_code_version=source_canvas.code_version
+ submitted_yaml,
+ inherited_code_version=source_canvas.code_version,
+ inherited_proxy_location=inherited_proxy_location,
)
== source_canvas
):
@@ -4016,6 +4024,7 @@ def _normalized_canvas_for_proposal_restore(
workflow_yaml: str,
*,
inherited_code_version: int | None = None,
+ inherited_proxy_location: ProxyLocationInput = None,
) -> WorkflowCreateYAMLRequest:
"""Normalize editor-only empty code outlines before comparing canvas custody.
@@ -4025,6 +4034,7 @@ def _normalized_canvas_for_proposal_restore(
"""
normalized = _normalize_copilot_yaml(workflow_yaml)
normalized.webhook_callback_url = normalized.webhook_callback_url or None
+ normalized.proxy_location = resolve_proxy_location(normalized, inherited_proxy_location)
normalized.extra_http_headers = None
normalized.cdp_connect_headers = None
normalized.mask_secrets = bool(normalized.mask_secrets)
@@ -4113,6 +4123,7 @@ async def _validated_pending_workflow_proposal(
proposal_workflow_run_id=metadata.workflow_run_id if metadata is not None else None,
)
return None
+ canonical: Workflow | None = None
if metadata is not None:
canonical = await app.DATABASE.workflows.get_workflow_by_permanent_id(
workflow_permanent_id=ctx.workflow_permanent_id,
@@ -4138,7 +4149,10 @@ async def _validated_pending_workflow_proposal(
# An explicit canvas edit remains the model's input to the normal update tool.
# Only restore over the persisted canvas or the same pending proposal.
if ctx.workflow_yaml and not copilot_workflow_yaml_matches(
- ctx.workflow_yaml, sanitized_yaml, ctx.persisted_workflow_yaml
+ ctx.workflow_yaml,
+ sanitized_yaml,
+ ctx.persisted_workflow_yaml,
+ inherited_proxy_location=canonical.proxy_location if canonical is not None else None,
):
LOG.info(
"copilot_pending_proposal_restore_rejected",
@@ -4149,6 +4163,42 @@ async def _validated_pending_workflow_proposal(
return proposal, sanitized_yaml
+_PROXY_LOCATION_ADAPTER = TypeAdapter(WorkflowCreateYAMLRequest.model_fields["proxy_location"].annotation)
+
+
+def _effective_private_setting(name: str, value: object) -> object:
+ # Accept reads a null header map as {} and an empty string as unset, and validates proxy spellings the way
+ # canvas custody does.
+ if name in ("extra_http_headers", "cdp_connect_headers"):
+ return value or {}
+ if name != "proxy_location":
+ return value or None
+ if value is None:
+ return None
+ canonical = {"proxy_location": value}
+ _canonicalize_copilot_proxy_location(canonical)
+ try:
+ return _PROXY_LOCATION_ADAPTER.validate_python(canonical["proxy_location"])
+ except ValidationError:
+ return value
+
+
+def _written_private_setting(name: str, value: object) -> object:
+ # Equal effective headers can still differ in written keys: an unbound mask is an explicit entry Accept resolves.
+ value = _effective_private_setting(name, value)
+ if isinstance(value, dict) and name in ("extra_http_headers", "cdp_connect_headers"):
+ return dict.fromkeys(value, SECRET_HEADER_MASK)
+ return value
+
+
+def _effective_private_settings(*sources: Mapping[str, object]) -> dict[str, object]:
+ # The first source that names a setting wins; Accept fills settings the proposal omits from the saved workflow.
+ return {
+ name: _effective_private_setting(name, next((source[name] for source in sources if name in source), None))
+ for name in _PRIVATE_WORKFLOW_SETTINGS_FIELDS
+ }
+
+
async def restore_pending_workflow_proposal(
ctx: CopilotContext,
*,
@@ -4182,21 +4232,18 @@ async def restore_pending_workflow_proposal(
saved = await app.DATABASE.workflows.get_workflow_by_permanent_id(
workflow_permanent_id=ctx.workflow_permanent_id, organization_id=ctx.organization_id
)
+ inherited_settings = inherited_header_settings(saved) if saved is not None else None
private_settings = merge_private_workflow_settings(
- proposal_settings,
- request_settings,
- inherited_settings=inherited_header_settings(saved) if saved is not None else None,
+ proposal_settings, request_settings, inherited_settings=inherited_settings
)
ctx.private_workflow_settings = private_settings
if pending is None:
return
_, sanitized_yaml = pending
metadata = copilot_proposal_metadata(proposal)
- restored_yaml = sanitized_yaml
- if private_settings:
- restored_document = safe_load_no_dates(sanitized_yaml)
- restored_document.update(private_settings)
- restored_yaml = dump_workflow_yaml(restored_document)
+ stored_document = safe_load_no_dates(sanitized_yaml)
+ restored_document = {**stored_document, **private_settings}
+ restored_yaml = dump_workflow_yaml(restored_document) if private_settings else sanitized_yaml
if "workflow_definition" in proposal:
workflow = Workflow.model_validate(proposal)
# A proposal froze its title in an earlier turn, so a rename saved between the two outranks
@@ -4238,7 +4285,39 @@ async def restore_pending_workflow_proposal(
settings_fallback_yaml=ctx.persisted_workflow_yaml,
prefer_live_title=True,
)
- workflow_yaml = strip_copilot_yaml_headers(restored_yaml)
+ # Serialization masks CDP headers, so the saved side takes its real header maps from the row itself.
+ saved_settings = (
+ {**submitted_private_workflow_settings(saved.model_dump(mode="json")), **inherited_header_settings(saved)}
+ if saved is not None
+ else {}
+ )
+ own_settings = merge_private_workflow_settings(proposal_settings, inherited_settings=inherited_settings)
+ own_effective = _effective_private_settings(own_settings, stored_document, saved_settings)
+ effective = _effective_private_settings(private_settings, stored_document, saved_settings)
+ if effective == own_effective:
+ # The turn-end publish compares settings as written, so a request value that only echoes the proposal's
+ # effective value is recorded as the proposal wrote it.
+ for name, value in request_settings.items():
+ if _written_private_setting(name, value) != _written_private_setting(name, own_effective.get(name)):
+ continue
+ if name in proposal_settings:
+ ctx.authored_private_workflow_settings[name] = copy.deepcopy(proposal_settings[name])
+ else:
+ ctx.authored_private_workflow_settings.pop(name, None)
+ elif metadata is not None:
+ ctx.settings_diverged_proposal_token = (metadata.owner_turn_id, metadata.revision)
+ LOG.info(
+ "copilot_pending_proposal_settings_diverged",
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ proposal_owner_turn_id=metadata.owner_turn_id,
+ proposal_revision=metadata.revision,
+ setting_names=sorted(name for name in own_effective if effective[name] != own_effective[name]),
+ )
+ strip_private_workflow_settings(own_effective)
+ strip_private_workflow_settings(effective)
+ # Run bind compares staged bytes to stored ones, and the model's next edit re-saves what it sees, so re-dump
+ # only when a visible setting changed.
+ workflow_yaml = sanitized_yaml if own_effective == effective else strip_copilot_yaml_headers(restored_yaml)
ctx.staged_workflow = workflow
ctx.staged_workflow_yaml = workflow_yaml
ctx.workflow_yaml = workflow_yaml
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 636ab01a5..593b17742 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -952,6 +952,7 @@ class WorkflowRunModel(Base):
start_fresh_browser = Column(Boolean, nullable=True)
reuse_browser_session = Column(Boolean, nullable=True)
reuse_bound_key = Column(String, nullable=True)
+ workflow_definition_sha256 = Column(String, nullable=True)
status = Column(String, nullable=False)
failure_reason = Column(String)
proxy_location = Column(String)
@@ -1928,6 +1929,15 @@ class CredentialModel(Base):
class DebugSessionModel(Base):
__tablename__ = "debug_sessions"
+ __table_args__ = (
+ Index(
+ "ix_debug_sessions_org_wpid_user_created_at",
+ "organization_id",
+ "workflow_permanent_id",
+ "user_id",
+ "created_at",
+ ),
+ )
debug_session_id = Column(String, primary_key=True, default=generate_debug_session_id)
organization_id = Column(String, nullable=False)
diff --git a/skyvern/forge/sdk/db/repositories/debug.py b/skyvern/forge/sdk/db/repositories/debug.py
index ed59819a7..5f51f4c12 100644
--- a/skyvern/forge/sdk/db/repositories/debug.py
+++ b/skyvern/forge/sdk/db/repositories/debug.py
@@ -34,6 +34,7 @@ class DebugRepository(BaseRepository):
.filter_by(deleted_at=None)
.filter_by(status="created")
.order_by(DebugSessionModel.created_at.desc())
+ .limit(1)
)
).first()
diff --git a/skyvern/forge/sdk/db/repositories/tasks.py b/skyvern/forge/sdk/db/repositories/tasks.py
index 671f11209..8879af7dc 100644
--- a/skyvern/forge/sdk/db/repositories/tasks.py
+++ b/skyvern/forge/sdk/db/repositories/tasks.py
@@ -265,38 +265,19 @@ class TasksRepository(BaseRepository):
total = (await session.execute(query)).scalar_one()
return float(total)
- @db_operation("get_workflow_run_progress_timestamps")
- async def get_workflow_run_progress_timestamps(
+ @db_operation("get_workflow_run_block_progress_timestamp")
+ async def get_workflow_run_block_progress_timestamp(
self,
workflow_run_id: str,
organization_id: str | None = None,
- ) -> tuple[datetime | None, datetime | None]:
- """Return ``(max(step.modified_at), max(workflow_run_block.modified_at))``
- for a given workflow run. Both values are scalar aggregates — no row
- hydration — and are designed for the copilot watchdog poll path where
- the only question is "has anything changed since the last poll?".
-
- Step updates are the per-LLM-call heartbeat (status transitions plus
- incremental token/cost accumulators — every ``update_step`` call bumps
- ``StepModel.modified_at``). Block updates cover non-task block types
- (CODE, TEXT_PROMPT) that never create a task row. Together the two
- aggregates cover every kind of block the copilot can run.
- """
+ ) -> datetime | None:
async with self.Session() as session:
- step_stmt = (
- select(func.max(StepModel.modified_at))
- .join(TaskModel, StepModel.task_id == TaskModel.task_id)
- .where(TaskModel.workflow_run_id == workflow_run_id)
- .where(StepModel.organization_id == organization_id)
- )
block_stmt = (
select(func.max(WorkflowRunBlockModel.modified_at))
.where(WorkflowRunBlockModel.workflow_run_id == workflow_run_id)
.where(WorkflowRunBlockModel.organization_id == organization_id)
)
- step_ts = (await session.execute(step_stmt)).scalar_one_or_none()
- block_ts = (await session.execute(block_stmt)).scalar_one_or_none()
- return step_ts, block_ts
+ return (await session.execute(block_stmt)).scalar_one_or_none()
@db_operation("get_total_unique_step_order_count_by_task_ids")
async def get_total_unique_step_order_count_by_task_ids(
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index f4053355e..76012af94 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -92,6 +92,7 @@ from skyvern.forge.sdk.workflow.constants import INTERIM_OUTPUT_SNAPSHOT_MAX_BYT
from skyvern.forge.sdk.workflow.credential_selection import clear_credential_selections_for_retry
from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter
from skyvern.forge.sdk.workflow.models.workflow import (
+ MIXED_RUN_DEFINITION_DIGEST,
WorkflowDefinition,
WorkflowRun,
WorkflowRunOutputParameter,
@@ -863,6 +864,7 @@ class WorkflowRunsRepository(BaseRepository):
copilot_session_id: str | None = None,
start_fresh_browser: bool | None = None,
created_by: str | None = None,
+ workflow_definition_sha256: str | None = None,
) -> WorkflowRun:
async with self.Session() as session:
kwargs: dict[str, Any] = {}
@@ -877,6 +879,7 @@ class WorkflowRunsRepository(BaseRepository):
start_fresh_browser=start_fresh_browser,
reuse_browser_session=reuse_browser_session,
reuse_bound_key=reuse_bound_key,
+ workflow_definition_sha256=workflow_definition_sha256,
proxy_location=serialize_proxy_location(proxy_location),
status="created",
webhook_callback_url=webhook_callback_url,
@@ -1220,6 +1223,7 @@ class WorkflowRunsRepository(BaseRepository):
only_from: Sequence[WorkflowRunStatus] | None = None,
job_id: str | None = None,
depends_on_workflow_run_id: str | None = None,
+ workflow_definition_sha256: str | None = None,
) -> WorkflowRun | None:
"""One conditional UPDATE to ``status`` from a non-terminal state (``only_from`` narrows it), so a late
cancel cannot clobber a finalization; None when the row was terminal or missing. Timestamps follow
@@ -1245,6 +1249,14 @@ class WorkflowRunsRepository(BaseRepository):
values["job_id"] = job_id
if depends_on_workflow_run_id:
values["depends_on_workflow_run_id"] = depends_on_workflow_run_id
+ if workflow_definition_sha256 is not None:
+ # Only invalidate: a retry re-enters with the row as it is now while earlier attempts' block rows still
+ # describe the old code, and a NULL digest (a run created before digests) cannot say what those ran.
+ stored = WorkflowRunModel.workflow_definition_sha256
+ values["workflow_definition_sha256"] = case(
+ (and_(stored.is_not(None), stored != workflow_definition_sha256), MIXED_RUN_DEFINITION_DIGEST),
+ else_=stored,
+ )
# The reopen/reset path clears attribution when it returns the row to `created`, so a
# later terminal transition starts from SQL NULL and this COALESCE fills the freshly
# derived document; on any row that already carries one it preserves the first writer.
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index 89311a53e..1acc826c9 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -700,6 +700,7 @@ def convert_to_workflow_run(
start_fresh_browser=workflow_run_model.start_fresh_browser,
reuse_browser_session=workflow_run_model.reuse_browser_session,
reuse_bound_key=workflow_run_model.reuse_bound_key,
+ workflow_definition_sha256=workflow_run_model.workflow_definition_sha256,
task_queue=workflow_run_model.task_queue,
status=WorkflowRunStatus[workflow_run_model.status],
failure_reason=workflow_run_model.failure_reason,
diff --git a/skyvern/forge/sdk/routes/credentials.py b/skyvern/forge/sdk/routes/credentials.py
index 8c5069ce3..8450566ce 100644
--- a/skyvern/forge/sdk/routes/credentials.py
+++ b/skyvern/forge/sdk/routes/credentials.py
@@ -704,14 +704,23 @@ async def create_credential(
current_org: Organization = Depends(org_auth_service.get_current_org_for_credential_routes),
current_user_id: str | None = Depends(org_auth_service.get_current_user_id_or_none),
) -> CredentialResponse:
+ credential_service = await prepare_credential_create(current_org.organization_id, data)
+ credential = await store_new_credential(
+ current_org.organization_id, data, current_user_id, credential_service, background_tasks
+ )
+ return _convert_to_response(credential)
+
+
+async def prepare_credential_create(organization_id: str, data: CreateCredentialRequest) -> CredentialVaultService:
+ """Every check a create runs before it touches a vault; raising here means nothing was stored."""
if isinstance(data.credential, NonEmptyPasswordCredential):
await _normalize_authenticator_totp_for_organization_or_raise(
data.credential,
- organization_id=current_org.organization_id,
+ organization_id=organization_id,
)
await app.AGENT_FUNCTION.validate_credential_write(
- organization_id=current_org.organization_id,
+ organization_id=organization_id,
credential_type=data.credential_type,
credential=data.credential,
existing_credential=None,
@@ -720,18 +729,26 @@ async def create_credential(
# Validate the profile link BEFORE provisioning, so an invalid/managed/cross-org/owned profile
# rejects with a 400 without leaving an orphan credential (and vault secret) behind.
if data.browser_profile_id is not None:
- await _validate_credential_browser_profile_id(data.browser_profile_id, current_org.organization_id)
+ await _validate_credential_browser_profile_id(data.browser_profile_id, organization_id)
- credential_service = await _get_credential_vault_service(
+ return await _get_credential_vault_service(
vault_type_override=data.vault_type,
- organization_id=current_org.organization_id,
+ organization_id=organization_id,
)
+
+async def store_new_credential(
+ organization_id: str,
+ data: CreateCredentialRequest,
+ created_by: str | None,
+ credential_service: CredentialVaultService,
+ background_tasks: BackgroundTasks,
+) -> Credential:
try:
credential = await credential_service.create_credential(
- organization_id=current_org.organization_id,
+ organization_id=organization_id,
data=data,
- created_by=current_user_id,
+ created_by=created_by,
)
except SkyvernHttpException as e:
detail = (
@@ -739,11 +756,9 @@ async def create_credential(
if e.error_message
else f"Custom credential service returned HTTP {e.status_code}"
)
- raise HTTPException(status_code=502, detail=detail)
+ raise HTTPException(status_code=502, detail=detail) from e
- await record_request_audit_event(
- current_org.organization_id, "credential.create", "credential", credential.credential_id
- )
+ await record_request_audit_event(organization_id, "credential.create", "credential", credential.credential_id)
if credential.vault_type == CredentialVaultType.BITWARDEN:
# Early resyncing the Bitwarden vault
background_tasks.add_task(fetch_credential_item_background, credential.item_id)
@@ -757,7 +772,7 @@ async def create_credential(
profile_update: dict[str, Any] = {"browser_profile_id": data.browser_profile_id} if bpid_provided else {}
credential = await _update_credential_or_profile_conflict(
credential_id=credential.credential_id,
- organization_id=current_org.organization_id,
+ organization_id=organization_id,
**profile_update,
# Only touch the pin when the caller sent it — supplying a profile alone must not reset it.
pin_saved_session_ip=data.pin_saved_session_ip if pin_provided else None,
@@ -766,14 +781,13 @@ async def create_credential(
background_tasks.add_task(
app.AGENT_FUNCTION.on_credential_saved,
- organization_id=current_org.organization_id,
+ organization_id=organization_id,
credential_id=credential.credential_id,
credential_type=data.credential_type,
- actor_user_id=current_user_id,
+ actor_user_id=created_by,
vault=credential.vault_type.value if credential.vault_type is not None else None,
)
-
- return _convert_to_response(credential)
+ return credential
LOGIN_TEST_PROMPT = (
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index f9faa3dd2..b9fd3ab47 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -15,7 +15,7 @@ from urllib.parse import urlparse
import structlog
import yaml
-from fastapi import Depends, File, Form, Header, HTTPException, Request, UploadFile, status
+from fastapi import BackgroundTasks, Depends, File, Form, Header, HTTPException, Request, UploadFile, status
from opentelemetry import trace as otel_trace
from pydantic import ValidationError
from sse_starlette import EventSourceResponse
@@ -23,6 +23,7 @@ from sse_starlette import EventSourceResponse
from skyvern import analytics
from skyvern.config import settings
from skyvern.constants import DEFAULT_WORKFLOW_TITLES
+from skyvern.exceptions import HttpException as VaultHttpException
from skyvern.exceptions import SkyvernHTTPException
from skyvern.forge import app
from skyvern.forge.agent_functions import AuditEvent
@@ -47,13 +48,17 @@ from skyvern.forge.sdk.copilot.context import (
clear_proposed_credential,
merge_approved_credentials_into_global_llm_context,
)
+from skyvern.forge.sdk.copilot.credential_generation import generate_registration_password
from skyvern.forge.sdk.copilot.credential_pause import (
+ MANUAL_SIGN_IN_CLAIM_SECONDS,
MANUAL_SIGN_IN_SAVE_TIMEOUT_SECONDS,
CredentialPauseRejection,
SignedInProfile,
check_credential_pause_resumable,
+ claim_credential_generation,
claim_manual_sign_in,
credential_pause_is_active,
+ finish_credential_generation,
finish_manual_sign_in,
pending_credential_requests,
resolve_credential_pause,
@@ -119,8 +124,16 @@ from skyvern.forge.sdk.routes.browser_profiles import (
_hard_delete_created_profile_after_store_failure,
create_profile_from_running_session,
)
+from skyvern.forge.sdk.routes.credentials import prepare_credential_create, store_new_credential
from skyvern.forge.sdk.routes.routers import base_router
from skyvern.forge.sdk.schemas.copilot_turn_outcome import PersistedCopilotComposerMode, ResponseKind, TurnOutcome
+from skyvern.forge.sdk.schemas.credentials import (
+ CreateCredentialRequest,
+ Credential,
+ CredentialType,
+ NonEmptyPasswordCredential,
+ PasswordCredential,
+)
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.workflow_copilot import (
COPILOT_PRIVATE_SETTINGS_KEY,
@@ -145,6 +158,8 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatSender,
WorkflowCopilotChatSummary,
WorkflowCopilotClearProposedWorkflowRequest,
+ WorkflowCopilotCredentialGenerateRequest,
+ WorkflowCopilotCredentialGenerateResult,
WorkflowCopilotCredentialResponseRequest,
WorkflowCopilotCredentialResponseResult,
WorkflowCopilotDisableAutoAcceptRequest,
@@ -163,6 +178,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
copilot_proposal_metadata,
)
from skyvern.forge.sdk.services import org_auth_service
+from skyvern.forge.sdk.services.credential.credential_vault_service import CredentialVaultService
from skyvern.forge.sdk.services.request_principal import get_request_principal
from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
from skyvern.forge.sdk.workflow.models.workflow import Workflow
@@ -1044,9 +1060,20 @@ async def _persist_proposed_workflow_state(
proposed_workflow_data = _build_proposed_workflow_data(updated_workflow, agent_result)
if agent_result.proposal_owner_turn_id is not None and agent_result.proposal_revision is not None:
stored_metadata = copilot_proposal_metadata(chat.proposed_workflow)
+ stored_yaml = (
+ chat.proposed_workflow.get("_copilot_yaml") if isinstance(chat.proposed_workflow, dict) else None
+ )
same_bytes = (
isinstance(chat.proposed_workflow, dict)
- and chat.proposed_workflow.get("_copilot_yaml") == proposed_workflow_data.get("_copilot_yaml")
+ and (
+ stored_yaml == proposed_workflow_data.get("_copilot_yaml")
+ # A rename saved since publication re-titles a turn that answered with the stored bytes;
+ # Accept resolves the rename.
+ or (
+ stored_yaml is not None
+ and stored_yaml == strip_copilot_yaml_headers(agent_result.workflow_yaml)
+ )
+ )
and private_workflow_settings_from_proposal(chat.proposed_workflow)
== private_workflow_settings_from_proposal(proposed_workflow_data)
)
@@ -1055,7 +1082,11 @@ async def _persist_proposed_workflow_state(
# markers. Those bytes carry the resolved title and Accept reparses them as they are.
proposed_workflow_data = {
**chat.proposed_workflow,
- **{key: value for key, value in proposed_workflow_data.items() if key.startswith("_copilot_")},
+ **{
+ key: value
+ for key, value in proposed_workflow_data.items()
+ if key.startswith("_copilot_") and key != "_copilot_yaml"
+ },
}
if _proposal_disposition(agent_result) != "review_untested":
# The marker is only ever added, so a candidate published untested and since
@@ -2667,6 +2698,8 @@ async def _new_copilot_chat_post(
detail="No pending proposal to test end to end.",
)
_apply_test_end_to_end_action(chat_request, validated_proposal["_copilot_yaml"])
+ # The Test action runs the proposal under its own settings, never the editor's.
+ submitted_private_settings = {}
await stream.send(
WorkflowCopilotProcessingUpdate(
@@ -4256,6 +4289,190 @@ async def workflow_copilot_credential_response(
return WorkflowCopilotCredentialResponseResult(result="accepted")
+_READBACK_PAGE_SIZE = 100
+
+
+async def _find_generated_credential(
+ organization_id: str,
+ data: CreateCredentialRequest,
+ created_by: str | None,
+ created_since: datetime,
+ credential_service: CredentialVaultService,
+) -> str | None:
+ """Return the one credential this card saved, or None when no single row holds the password it generated."""
+ credentials = await app.DATABASE.credentials.get_credentials(
+ organization_id, page_size=_READBACK_PAGE_SIZE, credential_type=CredentialType.PASSWORD, search=data.name
+ )
+
+ def created_in_window(credential: Credential) -> bool:
+ return credential.created_at.replace(tzinfo=credential.created_at.tzinfo or UTC) >= created_since
+
+ # Rows come newest first, so a full page whose oldest row is still in the window may hide more matches.
+ if len(credentials) >= _READBACK_PAGE_SIZE and created_in_window(credentials[-1]):
+ return None
+ matches = [
+ credential
+ for credential in credentials
+ if credential.name == data.name
+ and credential.username == data.credential.username
+ and credential.created_by == created_by
+ and created_in_window(credential)
+ ]
+ if len(matches) != 1:
+ return None
+ stored = (await credential_service.get_credential_item(matches[0])).credential
+ # Matching metadata alone could name another create of the same account, so only the generated password counts.
+ if isinstance(stored, PasswordCredential) and hmac.compare_digest(
+ stored.password.encode(), data.credential.password.encode()
+ ):
+ return matches[0].credential_id
+ return None
+
+
+def _log_late_create(stored: asyncio.Future[Credential]) -> None:
+ if not stored.cancelled() and stored.exception() is None:
+ LOG.warning("copilot_credential_generation_late_create", credential_id=stored.result().credential_id)
+
+
+_DETACHED_CREDENTIAL_CREATES: set[asyncio.Task[None]] = set()
+
+
+async def _create_generated_credential(
+ organization_id: str,
+ data: CreateCredentialRequest,
+ created_by: str | None,
+ credential_service: CredentialVaultService,
+ stored: asyncio.Future[Credential],
+) -> None:
+ # The request's own BackgroundTasks never run once it has answered, so a late save brings its own.
+ hooks = BackgroundTasks()
+ try:
+ credential = await store_new_credential(organization_id, data, created_by, credential_service, hooks)
+ except Exception as exc:
+ stored.set_exception(exc)
+ return
+ except asyncio.CancelledError:
+ stored.cancel()
+ raise
+ stored.set_result(credential)
+ # Nothing awaits this task, so a failing post-save hook would otherwise vanish as an unretrieved exception.
+ try:
+ await hooks()
+ except Exception:
+ LOG.warning("copilot_credential_generation_post_save_hooks_failed", exc_info=True)
+
+
+# Every wait leaves the claim the same margin to finish in that the manual sign-in save does.
+_CLAIM_FINISH_MARGIN_SECONDS = MANUAL_SIGN_IN_CLAIM_SECONDS - MANUAL_SIGN_IN_SAVE_TIMEOUT_SECONDS
+_READBACK_RESERVE_SECONDS = 15
+
+
+def _claim_seconds_left(deadline: datetime) -> float:
+ return max(0.0, (deadline - datetime.now(UTC)).total_seconds() - _CLAIM_FINISH_MARGIN_SECONDS)
+
+
+@base_router.post(
+ "/workflow/copilot/credential-generate",
+ include_in_schema=False,
+ response_model=WorkflowCopilotCredentialGenerateResult,
+)
+async def workflow_copilot_credential_generate(
+ generate_request: WorkflowCopilotCredentialGenerateRequest,
+ organization: Organization = Depends(org_auth_service.get_current_org_for_credential_routes),
+ current_user_id: str | None = Depends(org_auth_service.get_current_user_id_or_none),
+) -> WorkflowCopilotCredentialGenerateResult:
+ """Answer a registration card by generating its password here and saving it as a credential, once per card.
+
+ The password goes straight to the vault create: it is never returned, logged, or kept on the pause.
+ """
+ cache = getattr(app, "CACHE", None)
+ if cache is None:
+ raise HTTPException(
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+ detail="Credential response not supported in this environment",
+ )
+ organization_id = organization.organization_id
+ pause_ids = {
+ "organization_id": organization_id,
+ "workflow_copilot_chat_id": generate_request.workflow_copilot_chat_id,
+ "turn_id": generate_request.turn_id,
+ }
+ try:
+ claim = await claim_credential_generation(cache, resume_token=generate_request.resume_token, **pause_ids)
+ except CredentialPauseRejection as rejection:
+ raise HTTPException(status_code=rejection.status_code, detail=rejection.detail)
+ spec = claim.registration
+ claimed_at = datetime.now(UTC)
+ credential_id: str | None = None
+ outcome: Literal["rejected", "unknown"] = "unknown"
+ try:
+ password = generate_registration_password(spec.password_length, spec.charset)
+ # Registered before anything can raise or log with it, so the log chain scrubs it from tracebacks too.
+ skyvern_context.ensure_context().register_secret_value(password)
+ data = CreateCredentialRequest(
+ name=spec.credential_name,
+ credential_type=CredentialType.PASSWORD,
+ credential=NonEmptyPasswordCredential(username=spec.username, password=password),
+ )
+ credential_service = await asyncio.wait_for(
+ prepare_credential_create(organization_id, data), _claim_seconds_left(claim.deadline)
+ )
+ except Exception as exc:
+ outcome = "rejected"
+ LOG.warning("copilot_credential_generation_rejected", error_type=type(exc).__name__)
+ else:
+ # A vault create cancelled midway can skip its own cleanup, so a timeout stops waiting but lets it finish.
+ stored: asyncio.Future[Credential] = asyncio.get_running_loop().create_future()
+ stored.add_done_callback(lambda future: future.cancelled() or future.exception())
+ create = asyncio.create_task(
+ _create_generated_credential(organization_id, data, current_user_id, credential_service, stored)
+ )
+ _DETACHED_CREDENTIAL_CREATES.add(create)
+ create.add_done_callback(_DETACHED_CREDENTIAL_CREATES.discard)
+ create_wait = max(0.0, _claim_seconds_left(claim.deadline) - _READBACK_RESERVE_SECONDS)
+ try:
+ credential_id = (await asyncio.wait_for(asyncio.shield(stored), create_wait)).credential_id
+ except Exception as exc:
+ vault_error = exc.__cause__ if isinstance(exc, HTTPException) else None
+ if isinstance(vault_error, VaultHttpException) and 400 <= vault_error.status_code < 500:
+ outcome = "rejected"
+ LOG.warning("copilot_credential_generation_rejected", vault_status=vault_error.status_code)
+ else:
+ LOG.warning("copilot_credential_generation_unconfirmed", error_type=type(exc).__name__)
+ if credential_id is None and outcome == "unknown":
+ stored.add_done_callback(_log_late_create)
+ try:
+ credential_id = await asyncio.wait_for(
+ _find_generated_credential(organization_id, data, current_user_id, claimed_at, credential_service),
+ _claim_seconds_left(claim.deadline),
+ )
+ except Exception:
+ LOG.warning("copilot_credential_generation_readback_failed", exc_info=True)
+ LOG.info("copilot_credential_generation_readback", credential_id=credential_id)
+ try:
+ deadline = await finish_credential_generation(
+ cache, claim=claim, credential_id=credential_id, outcome=outcome, **pause_ids
+ )
+ except Exception:
+ if credential_id is None:
+ raise
+ LOG.warning("copilot_credential_generation_finish_failed", credential_id=credential_id, exc_info=True)
+ deadline = None
+ if credential_id is not None:
+ LOG.info("copilot_credential_generation_created", credential_id=credential_id, connected=bool(deadline))
+ return WorkflowCopilotCredentialGenerateResult(
+ result="connected" if deadline else "created_not_connected",
+ credential_id=credential_id,
+ name=spec.credential_name,
+ username=spec.username,
+ )
+ if deadline is None:
+ raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Credential pause already resolved")
+ return WorkflowCopilotCredentialGenerateResult(
+ result=outcome, name=spec.credential_name, username=spec.username, expires_at=deadline
+ )
+
+
@base_router.post(
"/workflow/copilot/clear-proposed-workflow", include_in_schema=False, status_code=status.HTTP_204_NO_CONTENT
)
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 4e1dbc1a6..5154ee9c0 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -455,6 +455,9 @@ class WorkflowCopilotChatRequest(BaseModel):
supports_credential_pause_recovery: bool = Field(
False, description="The client restores pending credential cards from chat history."
)
+ supports_credential_generation: bool = Field(
+ False, description="The client can render a registration card and post its Generate and save answer."
+ )
supports_credential_pause: bool = Field(
False,
description=(
@@ -542,6 +545,28 @@ class WorkflowCopilotCredentialResponseRequest(BaseModel):
credential_id: str | None = Field(None, description="Saved credential ID; required when action is 'connected'")
+class WorkflowCopilotCredentialGenerateRequest(BaseModel):
+ turn_id: str = Field(..., description="turn_id from the matching credential_required frame")
+ workflow_copilot_chat_id: str = Field(..., description="chat ID from the matching credential_required frame")
+ resume_token: str = Field(..., description="One-time resume token from the matching credential_required frame")
+
+
+class WorkflowCopilotCredentialGenerateResult(BaseModel):
+ result: Literal["connected", "rejected", "unknown", "created_not_connected"] = Field(
+ ...,
+ description=(
+ "'rejected' means nothing was saved; 'unknown' means the vault did not confirm the save, and the "
+ "card stays open without Generate"
+ ),
+ )
+ credential_id: str | None = Field(None, description="The saved credential, when one is known to exist")
+ name: str | None = None
+ username: str | None = None
+ expires_at: datetime | None = Field(
+ None, description="The reopened card's new deadline after 'rejected' or 'unknown'"
+ )
+
+
class WorkflowCopilotCredentialResponseResult(BaseModel):
result: Literal["accepted", "signed_in", "no_sign_in_found", "save_failed"]
expires_at: datetime | None = Field(None, description="The pause's new deadline after 'signing_in'")
@@ -961,6 +986,20 @@ class WorkflowCopilotTitleUpdate(BaseModel):
timestamp: datetime = Field(..., description="Server timestamp")
+class CredentialRegistration(BaseModel):
+ username: str = Field(..., description="Username or email to register with, exactly as the user gave it")
+ credential_name: str = Field(..., description="Name to save the new credential under")
+ password_length: int = Field(24, description="Generated password length; 24 to 128")
+ charset: Literal["alphanumeric", "alphanumeric_symbols"] = Field(
+ "alphanumeric_symbols", description="Characters the site accepts in a password"
+ )
+
+
+class WorkflowCopilotCredentialRegistration(CredentialRegistration):
+ attempted: bool = Field(False, description="Generate and save was used; the card offers it once")
+ outcome: Literal["rejected", "unknown"] | None = Field(None, description="Why an attempt did not connect")
+
+
class WorkflowCopilotCredentialRequiredUpdate(BaseModel):
type: WorkflowCopilotStreamMessageType = Field(
WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED, description="Message type"
@@ -975,6 +1014,7 @@ class WorkflowCopilotCredentialRequiredUpdate(BaseModel):
"login_credentials_unresolved",
"credential_missing_totp",
"credential_rejected_by_site",
+ "credential_registration",
] = Field(..., description="Typed signal that triggered the pause")
message: str = Field(..., description="The agent's explanatory text at the moment of pausing")
login_page_urls: list[str] = Field(default_factory=list, description="Candidate login page URLs, if known")
@@ -988,6 +1028,9 @@ class WorkflowCopilotCredentialRequiredUpdate(BaseModel):
None, description="The live browser the user may sign in to themselves; absent when the card does not offer it"
)
signing_in: bool = Field(False, description="The user has started signing in themselves")
+ registration: WorkflowCopilotCredentialRegistration | None = Field(
+ None, description="Account details a Generate and save answer would store; never a password"
+ )
timestamp: datetime = Field(..., description="Server timestamp")
diff --git a/skyvern/forge/sdk/services/route_authorization.py b/skyvern/forge/sdk/services/route_authorization.py
index b10d47d65..44008a75d 100644
--- a/skyvern/forge/sdk/services/route_authorization.py
+++ b/skyvern/forge/sdk/services/route_authorization.py
@@ -255,6 +255,7 @@ ROUTE_ACTIONS: Mapping[str, str] = {
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_chat_post": "workflow.update",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_clear_proposed_workflow": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_convert_yaml_to_blocks": "workflow_copilot.manage",
+ "skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_credential_generate": "credential.create",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_credential_response": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_disable_auto_accept": "workflow_copilot.manage",
"skyvern.forge.sdk.routes.workflow_copilot.workflow_copilot_message_feedback": "workflow_copilot.manage",
diff --git a/skyvern/forge/sdk/workflow/code_block_authorized_files.py b/skyvern/forge/sdk/workflow/code_block_authorized_files.py
index c67dbd72f..7ce8a3f4b 100644
--- a/skyvern/forge/sdk/workflow/code_block_authorized_files.py
+++ b/skyvern/forge/sdk/workflow/code_block_authorized_files.py
@@ -10,20 +10,22 @@ import re
import shutil
import stat
import tempfile
+import time
from collections.abc import Awaitable, Callable, Mapping
from contextlib import suppress
from dataclasses import dataclass
from functools import partial
from pathlib import Path, PurePath
from types import SimpleNamespace
-from typing import Any, Protocol, cast
+from typing import Any, Literal, Protocol, cast, get_args
import structlog
-from playwright.async_api import BrowserContext
+from playwright.async_api import BrowserContext, Page
from skyvern.constants import SAVE_DOWNLOADED_FILES_TIMEOUT
from skyvern.forge import app
from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.hashing import diagnostic_fingerprint
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.webeye.browser_factory import resolve_artifact_path
@@ -42,6 +44,20 @@ AUTHORIZED_FILE_CHANGED_ERROR = "The authorized file is unavailable or changed.
AUTHORIZED_FILE_SHAPE_ERROR = (
"attach_authorized_file accepts only a materialized file_url parameter or a download this run claimed."
)
+AUTHORIZED_FILE_MODE_ERROR = 'attach_authorized_file parameter \'mode\' must be "input" or "file_chooser".'
+AUTHORIZED_FILE_CHOOSER_UNAVAILABLE_ERROR = (
+ "File chooser mode is unavailable: this page or its browser worker does not support it, or the file is no "
+ "longer authorized for this run (phase=chooser, trigger_state=not_attempted)."
+)
+
+AuthorizedFileAttachMode = Literal["input", "file_chooser"]
+AuthorizedFileChooserPhase = Literal["chooser", "trigger", "attachment"]
+AuthorizedFileTriggerState = Literal["not_attempted", "confirmed", "unknown"]
+AUTHORIZED_FILE_ATTACH_MODES: tuple[AuthorizedFileAttachMode, ...] = get_args(AuthorizedFileAttachMode)
+# One bound for arming, the single click, the chooser wait and set_files.
+_CHOOSER_WAIT_SECONDS = 30
+# Ends the bound this far before the block deadline, so the typed failure returns before the block is cut off.
+_CHOOSER_DEADLINE_MARGIN_SECONDS = 1.0
LOG = structlog.get_logger()
@@ -58,14 +74,75 @@ class AuthorizedFileAccessError(Exception):
pass
-class FileInputTarget(Protocol):
- def locator(self, selector: str) -> FileInputLocator: ...
+class AuthorizedFileChooserError(AuthorizedFileAccessError):
+ def __init__(
+ self,
+ phase: AuthorizedFileChooserPhase,
+ trigger_state: AuthorizedFileTriggerState,
+ message: str | None = None,
+ ) -> None:
+ self.phase = phase
+ self.trigger_state = trigger_state
+ suffix = "" if trigger_state == "not_attempted" else "; the trigger is not clicked again"
+ super().__init__(
+ message or f"The file chooser attach failed (phase={phase}, trigger_state={trigger_state}){suffix}."
+ )
class FileInputLocator(Protocol):
async def set_input_files(self, files: dict[str, str | bytes]) -> None: ...
+class ChooserTrigger(Protocol):
+ async def click(self, *, timeout: float | None = None, trial: bool | None = None) -> None: ...
+
+
+class AttachLocator(FileInputLocator, ChooserTrigger, Protocol): ...
+
+
+class FileInputTarget(Protocol):
+ def locator(self, selector: str) -> AttachLocator: ...
+
+
+class InlineAttach(Protocol):
+ async def __call__(
+ self, page: FileInputTarget, file: str | DownloadLike, selector: str, mode: str = "input"
+ ) -> dict[str, str | int]: ...
+
+
+class FileChooserHandle(Protocol):
+ async def set_files(self, files: dict[str, str | bytes], *, timeout: float | None = None) -> None: ...
+
+
+@dataclass(frozen=True, slots=True)
+class FileChooserPins:
+ """Unrecorded locator and listener methods taken from the raw page's class before authored code runs."""
+
+ page: Page
+ locate: Callable[[str], ChooserTrigger]
+ listen: Callable[[str, Callable[[FileChooserHandle], None]], None]
+ unlisten: Callable[[str, Callable[[FileChooserHandle], None]], None]
+
+
+def authorized_file_attach_modes(raw_page: Page) -> frozenset[AuthorizedFileAttachMode]:
+ # The raw-CDP engine's FileChooser accepts host paths only, so it cannot take the verified in-memory bytes.
+ if type(raw_page).__module__.startswith(("playwright.", "patchright.")):
+ return frozenset(AUTHORIZED_FILE_ATTACH_MODES)
+ return frozenset({"input"})
+
+
+def pin_file_chooser(raw_page: Page) -> FileChooserPins | None:
+ if "file_chooser" not in authorized_file_attach_modes(raw_page):
+ return None
+ page_class = type(raw_page)
+ return FileChooserPins(
+ page=raw_page,
+ locate=partial(page_class.locator, raw_page),
+ listen=partial(page_class.on, raw_page),
+ unlisten=partial(page_class.remove_listener, raw_page),
+ )
+
+
@dataclass(frozen=True, slots=True)
class MaterializedAuthorizedFile:
workflow_run_id: str
@@ -768,11 +845,48 @@ async def attach_materialized_authorized_file(
) -> dict[str, str | int]:
# `locate` must be captured before authored code runs: inline code can reassign `page.locator`,
# and a target resolved through the live page would hand the verified bytes to that object.
+ _require_selector(selector)
+ target = locate(selector)
+ verified = await asyncio.to_thread(
+ read_authorized_file,
+ authorized_file,
+ download_root=download_root,
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ max_bytes=max_bytes,
+ )
+ await target.set_input_files(_file_payload(verified))
+ return {"filename": verified.filename, "size": verified.size}
+
+
+def _require_selector(selector: str) -> None:
if not isinstance(selector, str) or not selector:
raise AuthorizedFileAccessError(
"attach_authorized_file parameter 'selector' must be a non-empty selector string."
)
- target = locate(selector)
+
+
+def _file_payload(verified: AuthorizedFileBytes) -> dict[str, str | bytes]:
+ mime_type = mimetypes.guess_type(verified.filename)[0] or "application/octet-stream"
+ return {"name": verified.filename, "mimeType": mime_type, "buffer": verified.content}
+
+
+async def attach_authorized_file_via_chooser(
+ locate: Callable[[str], ChooserTrigger],
+ file_chooser: FileChooserPins | None,
+ authorized_file: MaterializedAuthorizedFile,
+ selector: str,
+ *,
+ download_root: str | Path,
+ workflow_run_id: str,
+ organization_id: str | None,
+ max_bytes: int,
+ deadline: float | None = None,
+ passes_through: Callable[[Exception], bool] | None = None,
+) -> dict[str, str | int]:
+ """Verify the file, click ``selector`` once and fill the chooser it opens, before the monotonic ``deadline``.
+ An error ``passes_through`` accepts is re-raised as is instead of becoming the typed chooser failure."""
+ _require_selector(selector)
verified = await asyncio.to_thread(
read_authorized_file,
authorized_file,
@@ -781,8 +895,64 @@ async def attach_materialized_authorized_file(
organization_id=organization_id,
max_bytes=max_bytes,
)
- mime_type = mimetypes.guess_type(verified.filename)[0] or "application/octet-stream"
- await target.set_input_files({"name": verified.filename, "mimeType": mime_type, "buffer": verified.content})
+ if file_chooser is None:
+ raise AuthorizedFileChooserError("chooser", "not_attempted", AUTHORIZED_FILE_CHOOSER_UNAVAILABLE_ERROR)
+ bound: float = _CHOOSER_WAIT_SECONDS
+ if deadline is not None:
+ bound = min(bound, deadline - time.monotonic() - _CHOOSER_DEADLINE_MARGIN_SECONDS)
+ if bound <= 0:
+ raise AuthorizedFileChooserError("trigger", "not_attempted")
+ # An agent UPLOAD_FILE that never saw its chooser leaves a listener on the page that would fill this one first.
+ context = skyvern_context.current()
+ if context is not None and context.pending_file_chooser is not None:
+ if context.pending_file_chooser.page is file_chooser.page:
+ context.cleanup_pending_file_chooser()
+ loop = asyncio.get_running_loop()
+ ends_at = loop.time() + bound
+ opened: asyncio.Future[FileChooserHandle] = loop.create_future()
+
+ def remaining_ms() -> float:
+ # Playwright keeps a call running after its task is cancelled, so each call carries the bound itself.
+ return max((ends_at - loop.time()) * 1000, 1)
+
+ phase: AuthorizedFileChooserPhase = "chooser"
+ trigger_state: AuthorizedFileTriggerState = "not_attempted"
+
+ def on_file_chooser(chooser: FileChooserHandle) -> None:
+ # A chooser the page opens before the real click is not the trigger's, so it never gets the bytes.
+ if trigger_state != "not_attempted" and not opened.done():
+ opened.set_result(chooser)
+
+ try:
+ async with asyncio.timeout_at(ends_at):
+ # A listener rather than expect_file_chooser: a cancelled Playwright waiter keeps its listener,
+ # and with it chooser interception, until its own timer fires.
+ file_chooser.listen("filechooser", on_file_chooser)
+ try:
+ phase = "trigger"
+ # Trial runs the actionability and strictness checks without dispatching, so a trigger
+ # that cannot be clicked fails before anything reaches the page.
+ await file_chooser.locate(selector).click(trial=True, timeout=remaining_ms())
+ trigger_state = "unknown"
+ await locate(selector).click(timeout=remaining_ms())
+ trigger_state = "confirmed"
+ phase = "chooser"
+ chooser = await opened
+ phase = "attachment"
+ await chooser.set_files(_file_payload(verified), timeout=remaining_ms())
+ finally:
+ with suppress(Exception):
+ file_chooser.unlisten("filechooser", on_file_chooser)
+ except Exception as exc:
+ if passes_through is not None and passes_through(exc):
+ raise
+ LOG.warning(
+ "codeblock.authorized_file_chooser_failed",
+ phase=phase,
+ trigger_state=trigger_state,
+ error_type=type(exc).__name__,
+ )
+ raise AuthorizedFileChooserError(phase, trigger_state) from None
return {"filename": verified.filename, "size": verified.size}
@@ -797,9 +967,11 @@ def bind_inline_attach_authorized_file(
max_bytes: int,
download_run_id: str | None = None,
download_log: BlockDownloadLog | None = None,
- locate: Callable[[str], FileInputLocator] | None = None,
+ locate: Callable[[str], AttachLocator] | None = None,
registered_downloads: RegisteredDownloadSource | None = None,
-) -> Callable[[FileInputTarget, str | DownloadLike, str], Awaitable[dict[str, str | int]]]:
+ file_chooser: FileChooserPins | None = None,
+ deadline: float | None = None,
+) -> InlineAttach:
authorized_by_path = {
inline_authorized_file_path(item, download_root=download_root): item
for item in materializations.values()
@@ -826,9 +998,12 @@ def bind_inline_attach_authorized_file(
page: FileInputTarget,
file: str | DownloadLike,
selector: str,
+ mode: str = "input",
) -> dict[str, str | int]:
if page is not expected_page:
raise AuthorizedFileAccessError("attach_authorized_file requires the current CodeBlock page.")
+ if type(mode) is not str or mode not in AUTHORIZED_FILE_ATTACH_MODES:
+ raise AuthorizedFileAccessError(AUTHORIZED_FILE_MODE_ERROR)
# Capability, not class identity: a raw-CDP download is not a Playwright ``Download``, and the
# bytes come from the run's own download directory either way, never from this object.
if is_download_like(file):
@@ -867,6 +1042,18 @@ def bind_inline_attach_authorized_file(
authorized_file = resolved
if locate is None:
raise AuthorizedFileAccessError("attach_authorized_file requires the current CodeBlock page.")
+ if mode == "file_chooser":
+ return await attach_authorized_file_via_chooser(
+ locate,
+ file_chooser,
+ authorized_file,
+ selector,
+ download_root=download_root,
+ workflow_run_id=run_key,
+ organization_id=organization_id,
+ max_bytes=max_bytes,
+ deadline=deadline,
+ )
return await attach_materialized_authorized_file(
locate,
authorized_file,
@@ -884,6 +1071,7 @@ async def unbound_attach_authorized_file(
page: FileInputTarget,
file: str,
selector: str,
+ mode: str = "input",
) -> dict[str, str | int]:
raise AuthorizedFileAccessError(
"attach_authorized_file is only available for a materialized file_url parameter during a workflow run."
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 343246dca..f58dddf79 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -208,6 +208,7 @@ from skyvern.forge.sdk.workflow.code_block_authorized_files import (
bind_inline_attach_authorized_file,
capture_authorized_file,
inline_authorized_file_path,
+ pin_file_chooser,
unbound_attach_authorized_file,
)
from skyvern.forge.sdk.workflow.code_block_safety import BLOCKED_ATTRS as CODE_BLOCK_BLOCKED_ATTRS
@@ -6787,6 +6788,13 @@ class CodeBlock(Block):
download_log=download_log,
locate=page._pinned_locator() if isinstance(page, RecordingPage) else None,
registered_downloads=registered_downloads,
+ file_chooser=pin_file_chooser(_raw_code_block_page(page)),
+ # Taken before execute_user_function_with_timeout starts its clock, so it errs early.
+ deadline=(
+ monotonic() + settings.CODE_BLOCK_EXECUTION_TIMEOUT_SECONDS
+ if settings.CODE_BLOCK_EXECUTION_TIMEOUT_SECONDS > 0
+ else None
+ ),
)
safe_vars["open_page"] = _bind_code_block_open_page(
page, opened_pages if opened_pages is not None else [], browser_state
@@ -9996,7 +10004,7 @@ async def wrapper({default_args}):
failure_page=failed_page,
workflow_run_context=workflow_run_context,
redaction_parameters=serialized_parameter_values,
- opened_pages=opened_pages,
+ opened_pages=[*opened_pages, *recording_page._claimed_popups()],
)
if inline_failure_page_state:
from skyvern.forge.sdk.workflow.models.code_block_recorder import append_failure_page_state
diff --git a/skyvern/forge/sdk/workflow/models/code_block_recorder.py b/skyvern/forge/sdk/workflow/models/code_block_recorder.py
index dcca11172..ebf09125b 100644
--- a/skyvern/forge/sdk/workflow/models/code_block_recorder.py
+++ b/skyvern/forge/sdk/workflow/models/code_block_recorder.py
@@ -462,6 +462,7 @@ class _Recorder:
# has to be the same object, or identity comparisons fail and the worker hands out a
# second handle for the page it already registered.
self.page_proxies: dict[int, tuple[Any, Any]] = {}
+ self.claimed_popups: dict[int, Page] = {}
self.failure_operation_generation = 0
self._next_action_order = 0
self._on_action = on_action
@@ -1244,6 +1245,13 @@ class RecordingPage:
def _wrap_page(self, page: Any) -> Any:
return self.__recording_page(page)
+ def __claim_popup(self, page: Page) -> RecordingPage:
+ # Playwright still yields a popup that closed before the block read it; it is wrapped but not
+ # claimed, so failure evidence never names a tab that is already gone.
+ if not page.is_closed():
+ self.__recorder.claimed_popups[id(page)] = page
+ return self.__recording_page(page)
+
def __recording_page(self, page: Any) -> Any:
if page is self.__page:
return self
@@ -1317,6 +1325,9 @@ class RecordingPage:
def failure_locator(self, exception: BaseException) -> Locator | None:
return self.__recorder.failed_locator if self.__recorder.failed_locator_exception is exception else None
+ def _claimed_popups(self) -> list[Page]:
+ return list(self.__recorder.claimed_popups.values())
+
def failure_page(self, exception: BaseException) -> Page | None:
"""The raw page the call that raised ``exception`` ran on, whether through the page, one of
its frames or a locator; None when the exception did not come from a recorded call."""
@@ -1499,9 +1510,7 @@ class RecordingPage:
generation,
self.__page,
event_value=(
- self.__recording_page
- if name == "expect_popup"
- else partial(_wrap_if_page, self.__recording_page)
+ self.__claim_popup if name == "expect_popup" else partial(_wrap_if_page, self.__recording_page)
),
)
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index 6c0a3d8c1..ee549d24a 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -1,3 +1,5 @@
+import hashlib
+import json
from datetime import datetime
from enum import StrEnum
from typing import Any, List
@@ -365,6 +367,8 @@ class WorkflowRun(BaseModel):
reuse_bound_key: str | None = Field(default=None, exclude=True)
# Internal routing: the worker queue the run was dispatched to. Never an API field.
task_queue: str | None = Field(default=None, exclude=True)
+ # Digest of the definition the run was created against; the saved row can be overwritten in place later.
+ workflow_definition_sha256: str | None = Field(default=None, exclude=True)
debug_session_id: str | None = None
status: WorkflowRunStatus
attempt: int = Field(default=1, description="One-based number of the current workflow run attempt")
@@ -436,6 +440,14 @@ class WorkflowRun(BaseModel):
return self.debug_session_id is not None
+# Stored when attempts of one run executed different definitions; it never equals a real digest.
+MIXED_RUN_DEFINITION_DIGEST = "mixed"
+
+
+def workflow_definition_sha256(definition: WorkflowDefinition) -> str:
+ return hashlib.sha256(json.dumps(definition.model_dump(mode="json"), sort_keys=True).encode("utf-8")).hexdigest()
+
+
def start_hold_reason(*, sequential_key: str | None, depends_on_workflow_run_id: str | None) -> str:
"""Why a run may wait by design before it starts: a sequential lane, a dependency, or neither."""
if sequential_key:
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index d7021ea72..74d35ece2 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -226,6 +226,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (
resolve_reuse_browser_session,
should_acquire_reused_session,
start_hold_reason,
+ workflow_definition_sha256,
)
from skyvern.forge.sdk.workflow.private_settings import (
resolve_cdp_connect_headers,
@@ -8127,7 +8128,12 @@ class WorkflowService:
# The run_with field records what the user requested (e.g. "code"),
# not whether a script was actually found. Execution mode is determined
# separately by is_script_run and script_mode below.
- await self.mark_workflow_run_as_running(workflow_run_id=workflow_run_id, run_with=workflow_run.run_with)
+ # The row can be rewritten while the run is queued, so re-stamp the digest from what executes.
+ await self.mark_workflow_run_as_running(
+ workflow_run_id=workflow_run_id,
+ run_with=workflow_run.run_with,
+ workflow_definition_sha256=workflow_definition_sha256(workflow.workflow_definition),
+ )
# Set script_mode on context so downstream code can skip expensive LLM calls
# Only enable when we actually have a script to run
@@ -11562,6 +11568,10 @@ class WorkflowService:
organization_id=organization_id,
)
+ if workflow is None:
+ workflow = await self.get_workflow(workflow_id=workflow_id)
+ definition_sha256 = workflow_definition_sha256(workflow.workflow_definition)
+
# Sample the kill switch exactly once, before request-time precedence. This persisted
# admission decision is authoritative: later flag changes stop new runs but never revoke an
# in-flight run or strip the forced-session fallback that was selected at admission.
@@ -11570,8 +11580,6 @@ class WorkflowService:
persisted_reuse_bound_key: str | None = None
persisted_reuse_browser_session = workflow_request.reuse_browser_session
if not browser_session_id:
- if workflow is None:
- workflow = await self.get_workflow(workflow_id=workflow_id)
configured_reuse = should_acquire_reused_session(
browser_session_id=None,
start_fresh_browser=workflow_request.start_fresh_browser,
@@ -11643,6 +11651,7 @@ class WorkflowService:
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
copilot_session_id=copilot_session_id,
created_by=created_by,
+ workflow_definition_sha256=definition_sha256,
)
# A block run creates its block-run rows only after setup, so the caller's intent
# is the only block-scoped signal enrolment can see here.
@@ -11717,6 +11726,7 @@ class WorkflowService:
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
copilot_session_id=copilot_session_id,
created_by=created_by,
+ workflow_definition_sha256=definition_sha256,
)
if not block_scoped:
await ensure_attempt_row(
@@ -13540,7 +13550,12 @@ class WorkflowService:
LOG.exception("Failed to cascade child entity status", workflow_run_id=workflow_run_id)
return workflow_run
- async def mark_workflow_run_as_running(self, workflow_run_id: str, run_with: str | None = None) -> WorkflowRun:
+ async def mark_workflow_run_as_running(
+ self,
+ workflow_run_id: str,
+ run_with: str | None = None,
+ workflow_definition_sha256: str | None = None,
+ ) -> WorkflowRun:
# Conditional UPDATE refuses to resurrect a finalized wr — prevents the
# cleanup cron from racing with re-entry paths and stomping timed_out
# back to running.
@@ -13548,6 +13563,7 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.running,
run_with=run_with,
+ workflow_definition_sha256=workflow_definition_sha256,
)
if workflow_run is None:
existing = await app.DATABASE.workflow_runs.get_workflow_run(
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index d36f2f4ba..8fd3d7270 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -61,7 +61,13 @@ from skyvern.forge.sdk.workflow.models.block import (
UrlBlock,
)
from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE, ContextParameter
-from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRequestBody, WorkflowRun, WorkflowRunStatus
+from skyvern.forge.sdk.workflow.models.workflow import (
+ Workflow,
+ WorkflowRequestBody,
+ WorkflowRun,
+ WorkflowRunStatus,
+ workflow_definition_sha256,
+)
from skyvern.schemas.proxy_location import runtime_proxy_location
from skyvern.schemas.runs import (
ProxyLocationInput,
@@ -809,9 +815,11 @@ async def run_task_v2_helper(
task_v2 = await app.DATABASE.observer.update_task_v2(
task_v2_id=task_v2_id, organization_id=organization_id, status=TaskV2Status.running
)
- await app.WORKFLOW_SERVICE.mark_workflow_run_as_running(workflow_run_id=workflow_run.workflow_run_id)
-
workflow = await app.WORKFLOW_SERVICE.get_workflow(workflow_id=workflow_run.workflow_id)
+ await app.WORKFLOW_SERVICE.mark_workflow_run_as_running(
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_definition_sha256=workflow_definition_sha256(workflow.workflow_definition),
+ )
await _set_up_workflow_context(workflow, workflow_run_id, organization)
user_prompt = task_v2.prompt
diff --git a/skyvern/webeye/skycdp/facade/artifacts.py b/skyvern/webeye/skycdp/facade/artifacts.py
index ef32000e6..751fbfdee 100644
--- a/skyvern/webeye/skycdp/facade/artifacts.py
+++ b/skyvern/webeye/skycdp/facade/artifacts.py
@@ -114,7 +114,8 @@ class FileChooser:
"""
return self._multiple
- async def set_files(self, files: str | list[str]) -> None:
+ async def set_files(self, files: str | list[str], *, timeout: float | None = None) -> None:
+ # Playwright's signature has `timeout` and production passes it; one CDP call has nothing to time out.
paths = [files] if isinstance(files, str) else list(files)
missing = [path for path in paths if not Path(path).is_file()]
if missing:
diff --git a/tests/conftest.py b/tests/conftest.py
index 151b28fac..ba2f7ecd1 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,17 +1,21 @@
"""Fixtures shared by every suite."""
import inspect
+import io
+import logging
from collections.abc import Callable, Iterator
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock
import pytest
+import structlog
-from skyvern.forge import app
+from skyvern.forge import app, forge_app_initializer, request_logging
from skyvern.forge.sdk.core import organization_age_cache
from skyvern.forge.sdk.experimentation.code_block_ai_fallback import CODE_BLOCK_AI_FALLBACK_FLAG
from skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProvider
+from skyvern.forge.sdk.forge_log import setup_logger
from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.services import organization_log_scope
@@ -26,6 +30,28 @@ def _isolate_organization_age_cache() -> Iterator[None]:
organization_log_scope._warmups_in_flight.clear()
+def rendered_log_stream(monkeypatch: pytest.MonkeyPatch) -> Iterator[io.StringIO]:
+ """Every log line as production renders it (JSON, raw request logging on), written to a stream."""
+ monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS", True)
+ monkeypatch.setattr(request_logging.settings, "JSON_LOGGING", True)
+ root_logger = logging.getLogger()
+ saved_handlers = root_logger.handlers[:]
+ saved_structlog_config = structlog.get_config()
+ setup_logger()
+ # create_api_app configures logging once per process; keep it from replacing this handler.
+ monkeypatch.setattr(forge_app_initializer, "_SERVER_LOGGING_CONFIGURED", True)
+ stream = io.StringIO()
+ handler = root_logger.handlers[0]
+ assert isinstance(handler, logging.StreamHandler)
+ handler.setStream(stream)
+ try:
+ yield stream
+ finally:
+ root_logger.handlers[:] = saved_handlers
+ structlog.configure(**saved_structlog_config)
+
+
class ForcedSinkFailure(RuntimeError):
"""Stands in for a real sink error when a test forces a side effect to fail."""
diff --git a/tests/unit/_file_chooser_fakes.py b/tests/unit/_file_chooser_fakes.py
new file mode 100644
index 000000000..049371e96
--- /dev/null
+++ b/tests/unit/_file_chooser_fakes.py
@@ -0,0 +1,126 @@
+"""A page whose upload button creates its file input only when clicked, so only a chooser listener sees it."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+
+from skyvern.forge.sdk.workflow.code_block_authorized_files import FileChooserHandle
+
+ChooserListener = Callable[[FileChooserHandle], None]
+
+
+async def _drive(timeout: float | None) -> None:
+ if timeout is None:
+ await asyncio.Event().wait()
+ else:
+ await asyncio.sleep(timeout / 1000)
+
+
+async def _outlive_cancel(page: FakeChooserPage, timeout: float | None) -> None:
+ # Like Playwright's driver: cancelling the caller leaves the call running until its own timeout.
+ driver_call = asyncio.ensure_future(_drive(timeout))
+ page.driver_calls.append(driver_call)
+ await asyncio.shield(driver_call)
+
+
+class FakeChooserHandle:
+ def __init__(self, page: FakeChooserPage, into: list[dict[str, str | bytes]] | None = None) -> None:
+ self._page = page
+ self._into = page.chosen if into is None else into
+
+ async def set_files(self, files: dict[str, str | bytes], *, timeout: float | None = None) -> None:
+ self._page.set_files_timeouts.append(timeout)
+ if self._page.set_files_outlives_cancel:
+ await _outlive_cancel(self._page, timeout)
+ if self._page.set_files_error is not None:
+ raise self._page.set_files_error
+ self._into.append(files)
+
+
+class FakeChooserTrigger:
+ def __init__(self, page: FakeChooserPage, selector: str) -> None:
+ self.page, self.selector = page, selector
+
+ async def click(self, *, timeout: float | None = None, trial: bool | None = None) -> None:
+ page = self.page
+ page.clicks.append((self.selector, bool(trial)))
+ page.click_timeouts.append(timeout)
+ if trial:
+ if page.trial_error is not None:
+ raise page.trial_error
+ if page.trial_opens_chooser:
+ for handler in list(page.listeners.get("filechooser", [])):
+ handler(FakeChooserHandle(page, into=page.stray_chosen))
+ return
+ if page.click_blocks:
+ await asyncio.Event().wait()
+ if page.click_outlives_cancel:
+ await _outlive_cancel(page, timeout)
+ if page.click_error is not None:
+ raise page.click_error
+ if page.opens_chooser:
+ for handler in list(page.listeners.get("filechooser", [])):
+ handler(FakeChooserHandle(page))
+
+ async def set_input_files(self, files: dict[str, str | bytes]) -> None:
+ self.page.input_files.append(files)
+
+
+class FakeChooserPage:
+ def __init__(
+ self,
+ *,
+ opens_chooser: bool = True,
+ trial_error: Exception | None = None,
+ click_error: Exception | None = None,
+ set_files_error: Exception | None = None,
+ click_blocks: bool = False,
+ click_outlives_cancel: bool = False,
+ set_files_outlives_cancel: bool = False,
+ trial_opens_chooser: bool = False,
+ ) -> None:
+ self.opens_chooser = opens_chooser
+ self.trial_error = trial_error
+ self.click_error = click_error
+ self.set_files_error = set_files_error
+ self.click_blocks = click_blocks
+ self.click_outlives_cancel = click_outlives_cancel
+ self.set_files_outlives_cancel = set_files_outlives_cancel
+ self.trial_opens_chooser = trial_opens_chooser
+ self.clicks: list[tuple[str, bool]] = []
+ self.click_timeouts: list[float | None] = []
+ self.set_files_timeouts: list[float | None] = []
+ self.driver_calls: list[asyncio.Future[None]] = []
+ self.chosen: list[dict[str, str | bytes]] = []
+ self.stray_chosen: list[dict[str, str | bytes]] = []
+ self.input_files: list[dict[str, str | bytes]] = []
+ self.listeners: dict[str, list[ChooserListener]] = {}
+
+ def locator(self, selector: str) -> FakeChooserTrigger:
+ return FakeChooserTrigger(self, selector)
+
+ def on(self, event: str, handler: ChooserListener) -> None:
+ self.listeners.setdefault(event, []).append(handler)
+
+ def remove_listener(self, event: str, handler: ChooserListener) -> None:
+ self.listeners[event].remove(handler)
+
+ @property
+ def dispatched_clicks(self) -> int:
+ return sum(1 for _, trial in self.clicks if not trial)
+
+ @property
+ def live_listeners(self) -> int:
+ return len(self.listeners.get("filechooser", []))
+
+
+# Chooser mode is offered by page class family, so the fakes claim the module of the page they stand in for.
+FakeChooserPage.__module__ = "playwright.async_api._generated"
+
+
+class FakeRawCDPChooserPage(FakeChooserPage):
+ pass
+
+
+FakeRawCDPChooserPage.__module__ = "skyvern.webeye.skycdp.facade.page"
diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.py
index 1135be0ac..279a3c4ed 100644
--- a/tests/unit/copilot_test_helpers.py
+++ b/tests/unit/copilot_test_helpers.py
@@ -57,9 +57,11 @@ from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.persistent_browser_sessions import PersistentBrowserSession
from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest, WorkflowCopilotTitleUpdate
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
+from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, WorkflowParameter, WorkflowParameterType
from skyvern.forge.sdk.workflow.models.workflow import (
Workflow,
+ WorkflowDefinition,
WorkflowRun,
WorkflowRunOutputParameter,
WorkflowRunParameter,
@@ -236,6 +238,7 @@ def harness_run(
copilot_session_id: str | None = None,
workflow_permanent_id: str = "wpid-1",
organization_id: str = "org-1",
+ workflow_definition_sha256: str | None = None,
) -> SimpleNamespace:
"""Naive-UTC created_at, as the database returns it."""
return SimpleNamespace(
@@ -249,6 +252,7 @@ def harness_run(
copilot_session_id=copilot_session_id,
failure_reason=None,
browser_session_id="pbs-1",
+ workflow_definition_sha256=workflow_definition_sha256,
)
@@ -266,14 +270,23 @@ def install_get_run_results_harness(
carried_successful_run_id: str | None = None,
carried_run_id: str | None = None,
heal_episodes: list[HealEpisode] | None = None,
+ run_definition_sha256: str | None = None,
) -> SimpleNamespace:
"""Stub the collaborators ``_get_run_results`` reaches and return the ctx to call it with; the run pool is
``wr-1`` plus ``other_runs``, and run lookup and history listing honor their arguments."""
- pool = [harness_run("wr-1", status=run_status), *(other_runs or [])]
+ pool = [
+ harness_run("wr-1", status=run_status, workflow_definition_sha256=run_definition_sha256),
+ *(other_runs or []),
+ ]
workflow = SimpleNamespace(
+ workflow_id="wf-1",
+ workflow_permanent_id="wpid-1",
+ version=1,
created_by=None,
modified_at=HARNESS_RUN_CREATED_AT,
- workflow_definition=SimpleNamespace(parameters=workflow_parameters or [], blocks=[]),
+ workflow_definition=SimpleNamespace(
+ parameters=[SimpleNamespace(**parameter) for parameter in workflow_parameters or []], blocks=[]
+ ),
)
async def get_workflow_run(workflow_run_id: str, organization_id: str | None = None) -> SimpleNamespace | None:
@@ -349,7 +362,86 @@ def install_get_run_results_harness(
proposal_workflow_run_id=None,
dispatched_run_ids_this_turn=set(),
seeded_only_labels_by_run_id={},
+ workflow_yaml=None,
+ staged_workflow_yaml=None,
+ )
+
+
+def historical_code_version(
+ blocks: Mapping[str, str],
+ *,
+ parameter_keys: Sequence[str] = ("applicant_name",),
+ default_value: str = "Fixture default",
+ workflow_id: str = "wf-1",
+ modified_at: datetime = HARNESS_RUN_CREATED_AT,
+) -> SimpleNamespace:
+ """A saved version of ``wpid-1`` holding one CodeBlock per ``blocks`` label, as a finished harness run read it."""
+ return SimpleNamespace(
+ workflow_id=workflow_id,
+ workflow_permanent_id="wpid-1",
+ version=3,
+ created_by=None,
+ modified_at=modified_at,
+ workflow_definition=WorkflowDefinition(
+ parameters=[
+ WorkflowParameter(
+ workflow_parameter_id=f"wp_{key}",
+ workflow_parameter_type=WorkflowParameterType.STRING,
+ key=key,
+ default_value=default_value,
+ workflow_id=workflow_id,
+ created_at=HARNESS_RUN_CREATED_AT,
+ modified_at=HARNESS_RUN_CREATED_AT,
+ )
+ for key in parameter_keys
+ ],
+ blocks=[
+ CodeBlock(
+ label=label,
+ code=code,
+ output_parameter=OutputParameter(
+ output_parameter_id=f"op_{label}",
+ key=f"{label}_output",
+ workflow_id=workflow_id,
+ created_at=HARNESS_RUN_CREATED_AT,
+ modified_at=HARNESS_RUN_CREATED_AT,
+ ),
+ )
+ for label, code in blocks.items()
+ ],
+ ),
+ )
+
+
+def install_historical_run(
+ monkeypatch: pytest.MonkeyPatch,
+ version: SimpleNamespace | None,
+ *,
+ failed_label: str,
+ failing_line: int = 1,
+ run_definition_sha256: str | None = None,
+) -> None:
+ """Harness run ``wr-1`` failed at ``failing_line`` of ``failed_label`` while executing ``version``."""
+
+ async def stamp_failing_line(
+ _rows: object, results: list[dict[str, Any]], _org: str, include_completed: bool = False
+ ) -> None:
+ results[0]["action_trace"] = [{"action": "NULL_ACTION", "status": "failed", "code_line": failing_line}]
+
+ block = run_result_block_row(
+ failed_label, "failed", failure_reason="NameError: a name is not defined", error_codes=["user_code_error"]
+ )
+ block.created_at = HARNESS_RUN_CREATED_AT
+ block.parent_workflow_run_block_id = None
+ block.current_index = None
+ block.current_value = None
+ install_get_run_results_harness(
+ monkeypatch,
+ blocks=[block],
+ attach_action_traces=stamp_failing_line,
+ run_definition_sha256=run_definition_sha256,
)
+ run_execution_module.app.DATABASE.workflows.get_workflow_for_workflow_run.return_value = version
def run_result_action_row(
@@ -490,8 +582,8 @@ async def install_run_blocks_harness(
polled_run = _fake_workflow_run(status=polled_status)
- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, Any, Any]:
- return polled_run, now, now
+ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, Any]:
+ return polled_run, now
monkeypatch.setattr(run_execution_module, "_read_progress_sources", _read_progress)
monkeypatch.setattr(run_execution_module, "RUN_BLOCKS_POLL_INTERVAL_SECONDS", 0)
diff --git a/tests/unit/forge/sdk/db/test_workflow_attribution_columns.py b/tests/unit/forge/sdk/db/test_workflow_attribution_columns.py
index c9b164a7b..a1702102a 100644
--- a/tests/unit/forge/sdk/db/test_workflow_attribution_columns.py
+++ b/tests/unit/forge/sdk/db/test_workflow_attribution_columns.py
@@ -11,6 +11,11 @@ from sqlalchemy.ext.asyncio import create_async_engine
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.db.agent_db import AgentDB
from skyvern.forge.sdk.db.models import Base
+from skyvern.forge.sdk.workflow.models.workflow import (
+ MIXED_RUN_DEFINITION_DIGEST,
+ WorkflowRunStatus,
+ workflow_definition_sha256,
+)
@pytest_asyncio.fixture
@@ -218,3 +223,104 @@ async def test_workflow_never_touched_by_copilot_is_not_copilot_authored(agent_d
workflow_permanent_id=workflow.workflow_permanent_id,
organization_id=org_id,
)
+
+
+async def test_run_definition_digest_survives_reload_and_metadata_edits_but_not_a_rewrite(
+ agent_db: AgentDB, org_id: str
+) -> None:
+ workflow = await agent_db.workflows.create_workflow(
+ title="wf",
+ workflow_definition={"parameters": [], "blocks": []},
+ organization_id=org_id,
+ )
+ run = await agent_db.workflow_runs.create_workflow_run(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_id=workflow.workflow_id,
+ organization_id=org_id,
+ workflow_definition_sha256=workflow_definition_sha256(workflow.workflow_definition),
+ )
+ stored = await agent_db.workflow_runs.get_workflow_run(run.workflow_run_id, organization_id=org_id)
+ assert stored is not None and stored.workflow_definition_sha256 is not None
+
+ async def digest_of_row_the_run_reads() -> str:
+ row = await agent_db.workflows.get_workflow_for_workflow_run(run.workflow_run_id, organization_id=org_id)
+ assert row is not None
+ return workflow_definition_sha256(row.workflow_definition)
+
+ assert await digest_of_row_the_run_reads() == stored.workflow_definition_sha256
+ await agent_db.workflows.update_workflow(workflow_id=workflow.workflow_id, organization_id=org_id, title="renamed")
+ assert await digest_of_row_the_run_reads() == stored.workflow_definition_sha256
+ await agent_db.workflows.update_workflow(
+ workflow_id=workflow.workflow_id,
+ organization_id=org_id,
+ workflow_definition={"parameters": [], "blocks": [], "workflow_system_prompt": "rewritten in place"},
+ )
+ assert await digest_of_row_the_run_reads() != stored.workflow_definition_sha256
+
+
+ "first_attempt_runs_original", [True, False], ids=["rewritten-between-retries", "rewritten-while-queued"]
+)
+async def test_a_run_whose_attempts_saw_different_definitions_never_matches_its_row(
+ agent_db: AgentDB, org_id: str, first_attempt_runs_original: bool
+) -> None:
+ original = {"parameters": [], "blocks": []}
+ workflow = await agent_db.workflows.create_workflow(
+ title="wf", workflow_definition=original, organization_id=org_id
+ )
+ original_digest = workflow_definition_sha256(workflow.workflow_definition)
+ run = await agent_db.workflow_runs.create_workflow_run(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_id=workflow.workflow_id,
+ organization_id=org_id,
+ workflow_definition_sha256=original_digest,
+ )
+
+ async def start_attempt(digest: str) -> str | None:
+ started = await agent_db.workflow_runs.update_workflow_run_if_not_final(
+ workflow_run_id=run.workflow_run_id, status=WorkflowRunStatus.running, workflow_definition_sha256=digest
+ )
+ assert started is not None
+ return started.workflow_definition_sha256
+
+ if first_attempt_runs_original:
+ assert await start_attempt(original_digest) == original_digest
+ rewritten = await agent_db.workflows.update_workflow(
+ workflow_id=workflow.workflow_id,
+ organization_id=org_id,
+ workflow_definition={**original, "workflow_system_prompt": "rewritten in place"},
+ )
+ stored = await start_attempt(workflow_definition_sha256(rewritten.workflow_definition))
+
+ row = await agent_db.workflows.get_workflow_for_workflow_run(run.workflow_run_id, organization_id=org_id)
+ assert row is not None
+ assert stored == MIXED_RUN_DEFINITION_DIGEST
+ assert workflow_definition_sha256(row.workflow_definition) != stored
+
+
+async def test_a_run_created_before_digests_is_never_stamped_by_a_later_attempt(agent_db: AgentDB, org_id: str) -> None:
+ # Its earlier attempts' block rows may describe a definition nobody recorded, so it stays on the modified_at rule.
+ workflow = await agent_db.workflows.create_workflow(
+ title="wf", workflow_definition={"parameters": [], "blocks": []}, organization_id=org_id
+ )
+ run = await agent_db.workflow_runs.create_workflow_run(
+ workflow_permanent_id=workflow.workflow_permanent_id,
+ workflow_id=workflow.workflow_id,
+ organization_id=org_id,
+ )
+ rewritten = await agent_db.workflows.update_workflow(
+ workflow_id=workflow.workflow_id,
+ organization_id=org_id,
+ workflow_definition={"parameters": [], "blocks": [], "workflow_system_prompt": "rewritten before the retry"},
+ )
+
+ retried = await agent_db.workflow_runs.update_workflow_run_if_not_final(
+ workflow_run_id=run.workflow_run_id,
+ status=WorkflowRunStatus.running,
+ workflow_definition_sha256=workflow_definition_sha256(rewritten.workflow_definition),
+ )
+
+ assert retried is not None and retried.workflow_definition_sha256 is None
diff --git a/tests/unit/test_code_block_authorized_files.py b/tests/unit/test_code_block_authorized_files.py
index 2e7eab1fd..a2281838e 100644
--- a/tests/unit/test_code_block_authorized_files.py
+++ b/tests/unit/test_code_block_authorized_files.py
@@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from playwright.async_api import Download
+from playwright.async_api import Error as PlaywrightError
from structlog.testing import capture_logs
from skyvern.config import settings
@@ -23,17 +24,26 @@ from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.artifact.storage import base as storage_base_module
from skyvern.forge.sdk.artifact.storage import local as local_storage_module
from skyvern.forge.sdk.artifact.storage.local import LocalStorage
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import PendingFileChooserListener, SkyvernContext
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.workflow import code_block_authorized_files
from skyvern.forge.sdk.workflow.code_block_authorized_files import (
+ AUTHORIZED_FILE_CHOOSER_UNAVAILABLE_ERROR,
AuthorizedFileAccessError,
+ AuthorizedFileChooserError,
AuthorizedFileMaterializationFailure,
BlockDownloadLog,
+ FileChooserHandle,
+ InlineAttach,
+ MaterializedAuthorizedFile,
RegisteredDownloadIdentity,
RegisteredDownloadSource,
+ authorized_file_attach_modes,
bind_inline_attach_authorized_file,
capture_authorized_file,
capture_claimed_download,
+ pin_file_chooser,
read_authorized_file,
)
from skyvern.forge.sdk.workflow.models.block import CodeBlock, _registered_download_source
@@ -42,6 +52,7 @@ from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, Paramet
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.browser_artifacts import DownloadBinding
from skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptor
+from tests.unit._file_chooser_fakes import FakeChooserPage, FakeRawCDPChooserPage
from tests.unit.conftest import SESSION_DOWNLOAD_BYTES as _SESSION_BYTES
from tests.unit.conftest import make_claimed_download_mock
from tests.unit.conftest import registered_download_row as _registered_row
@@ -1606,3 +1617,377 @@ async def test_only_a_signed_remote_binding_snapshots_the_registered_downloads(m
("certificate.pdf", hashlib.sha256(_SESSION_BYTES).hexdigest()): "certificate.pdf"
}
assert probe.await_count == 1
+
+
+def _bind_chooser_attach(
+ raw_page: FakeChooserPage,
+ path: Path,
+ materialized: MaterializedAuthorizedFile,
+ *,
+ workflow_run_id: str = "wr_authorized",
+ organization_id: str = "org_authorized",
+ max_bytes: int = 1024,
+ deadline: float | None = None,
+) -> tuple[RecordingPage, InlineAttach]:
+ recording_page = RecordingPage(raw_page) # type: ignore[arg-type]
+ attach = bind_inline_attach_authorized_file(
+ recording_page, # type: ignore[arg-type]
+ {"resume": materialized, "cover_letter": AuthorizedFileMaterializationFailure()},
+ {"resume": str(path), "cover_letter": "https://invalid.example/cover.pdf"},
+ download_root=path.parents[1],
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ max_bytes=max_bytes,
+ locate=recording_page._pinned_locator(),
+ file_chooser=pin_file_chooser(raw_page),
+ deadline=deadline,
+ )
+ return recording_page, attach
+
+
+def _recorded_types(recording_page: RecordingPage) -> list[ActionType]:
+ return [action.action_type for action in recording_page.recorded_actions()]
+
+
+async def test_chooser_mode_clicks_the_trigger_once_and_hands_the_verified_bytes_to_the_chooser(
+ authorized_file,
+) -> None:
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage()
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)
+
+ receipt = await attach(recording_page, str(path), "#choose-file", mode="file_chooser")
+
+ assert receipt == {"filename": "resume.pdf", "size": len(b"retained authorized bytes")}
+ assert raw_page.clicks == [("#choose-file", True), ("#choose-file", False)]
+ assert [files["buffer"] for files in raw_page.chosen] == [b"retained authorized bytes"]
+ assert raw_page.input_files == []
+ # The trial click goes through the unrecorded raw locator, so the timeline shows the one real click.
+ assert _recorded_types(recording_page) == [ActionType.CLICK]
+ assert raw_page.live_listeners == 0
+
+
+async def test_a_chooser_the_page_opens_before_the_real_click_never_receives_the_bytes(authorized_file) -> None:
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage(trial_opens_chooser=True)
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)
+
+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")
+
+ assert raw_page.stray_chosen == []
+ assert [files["buffer"] for files in raw_page.chosen] == [b"retained authorized bytes"]
+ assert raw_page.dispatched_clicks == 1
+
+
+ ("page_options", "phase", "trigger_state", "dispatched", "recorded"),
+ [
+ ({"trial_error": PlaywrightError("strict mode violation: 2 elements")}, "trigger", "not_attempted", 0, []),
+ ({"click_error": PlaywrightError("element was detached")}, "trigger", "unknown", 1, [ActionType.CLICK]),
+ ({"opens_chooser": False}, "chooser", "confirmed", 1, [ActionType.CLICK]),
+ ({"set_files_error": PlaywrightError("chooser closed")}, "attachment", "confirmed", 1, [ActionType.CLICK]),
+ ],
+ ids=["trial-fails", "click-raises", "no-chooser", "set-files-fails"],
+)
+async def test_a_chooser_failure_names_its_phase_and_never_clicks_twice(
+ authorized_file,
+ monkeypatch: pytest.MonkeyPatch,
+ page_options: dict[str, Exception | bool],
+ phase: str,
+ trigger_state: str,
+ dispatched: int,
+ recorded: list[ActionType],
+) -> None:
+ monkeypatch.setattr(code_block_authorized_files, "_CHOOSER_WAIT_SECONDS", 0.2)
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage(**page_options)
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)
+
+ started = time.monotonic()
+ with pytest.raises(AuthorizedFileChooserError) as failure:
+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")
+
+ assert time.monotonic() - started < 2
+ assert (failure.value.phase, failure.value.trigger_state) == (phase, trigger_state)
+ assert failure.value.__cause__ is None
+ assert "retained authorized bytes" not in str(failure.value)
+ assert raw_page.dispatched_clicks == dispatched
+ assert _recorded_types(recording_page) == recorded
+ assert raw_page.chosen == [] and raw_page.input_files == []
+ assert raw_page.live_listeners == 0
+
+
+async def test_a_timed_out_chooser_leaves_nothing_for_the_next_attach_to_collect(
+ authorized_file, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(code_block_authorized_files, "_CHOOSER_WAIT_SECONDS", 0.2)
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage(opens_chooser=False)
+ first_page, first_attach = _bind_chooser_attach(raw_page, path, materialized)
+ with pytest.raises(AuthorizedFileChooserError):
+ await first_attach(first_page, str(path), "#choose-file", mode="file_chooser")
+ assert raw_page.live_listeners == 0
+
+ raw_page.opens_chooser = True
+ second_page, second_attach = _bind_chooser_attach(raw_page, path, materialized)
+ await second_attach(second_page, str(path), "#choose-file", mode="file_chooser")
+
+ assert len(raw_page.chosen) == 1
+ assert raw_page.dispatched_clicks == 2
+ assert raw_page.live_listeners == 0
+
+
+async def test_cancelling_a_chooser_attach_propagates_and_removes_its_listener(authorized_file) -> None:
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage(click_blocks=True)
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)
+
+ task = asyncio.create_task(attach(recording_page, str(path), "#choose-file", mode="file_chooser"))
+ while raw_page.dispatched_clicks == 0:
+ await asyncio.sleep(0)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ assert raw_page.live_listeners == 0
+ assert raw_page.dispatched_clicks == 1
+ assert raw_page.chosen == []
+
+
+ ("set_files_stalls", "phase", "trigger_state", "set_files_calls"),
+ [(False, "trigger", "unknown", 0), (True, "attachment", "confirmed", 1)],
+ ids=["click", "set-files"],
+)
+async def test_a_chooser_call_cut_off_by_the_bound_stops_in_the_driver_too(
+ authorized_file,
+ monkeypatch: pytest.MonkeyPatch,
+ set_files_stalls: bool,
+ phase: str,
+ trigger_state: str,
+ set_files_calls: int,
+) -> None:
+ """A call whose caller is cancelled keeps running, so only its own timeout keeps it inside the bound."""
+ monkeypatch.setattr(code_block_authorized_files, "_CHOOSER_WAIT_SECONDS", 0.2)
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage(click_outlives_cancel=not set_files_stalls, set_files_outlives_cancel=set_files_stalls)
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)
+
+ with pytest.raises(AuthorizedFileChooserError) as failure:
+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")
+
+ assert (failure.value.phase, failure.value.trigger_state) == (phase, trigger_state)
+ assert [trial for _, trial in raw_page.clicks] == [True, False]
+ assert len(raw_page.set_files_timeouts) == set_files_calls
+ timeouts = raw_page.click_timeouts + raw_page.set_files_timeouts
+ assert all(timeout is not None and 1 <= timeout <= 200 for timeout in timeouts)
+ _, still_running = await asyncio.wait(raw_page.driver_calls, timeout=1)
+ assert still_running == set()
+
+
+async def test_a_chooser_attach_leaves_an_agent_upload_listener_on_another_page_alone(authorized_file) -> None:
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage()
+ other_page = FakeChooserPage()
+ other_chosen: list[FileChooserHandle] = []
+ other_handler = other_chosen.append
+ other_page.on("filechooser", other_handler)
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)
+
+ with skyvern_context.scoped(SkyvernContext()) as context:
+ pending = PendingFileChooserListener(
+ page=other_page, # type: ignore[arg-type]
+ file_paths=["/tmp/agent-upload.pdf"],
+ handler=other_handler,
+ )
+ context.pending_file_chooser = pending
+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")
+ assert context.pending_file_chooser is pending
+ assert other_page.listeners["filechooser"] == [other_handler]
+
+ assert [files["buffer"] for files in raw_page.chosen] == [b"retained authorized bytes"]
+
+
+ ("file", "selector", "mode", "bind_overrides", "tamper"),
+ [
+ ("resume", "#choose-file", "chooser", {}, False),
+ ("resume", "#choose-file", "File_Chooser", {}, False),
+ ("raw-path", "#choose-file", "file_chooser", {}, False),
+ ("https://invalid.example/cover.pdf", "#choose-file", "file_chooser", {}, False),
+ ("resume", "", "file_chooser", {}, False),
+ ("resume", "#choose-file", "file_chooser", {"workflow_run_id": "wr_other"}, False),
+ ("resume", "#choose-file", "file_chooser", {"organization_id": "org_other"}, False),
+ ("resume", "#choose-file", "file_chooser", {"max_bytes": 4}, False),
+ ("resume", "#choose-file", "file_chooser", {}, True),
+ ],
+ ids=[
+ "unknown-mode",
+ "mode-is-exact",
+ "raw-path",
+ "unmaterialized",
+ "empty-selector",
+ "wrong-run",
+ "wrong-org",
+ "oversized",
+ "tampered",
+ ],
+)
+async def test_a_refused_chooser_attach_never_clicks_or_hands_over_bytes(
+ authorized_file,
+ tmp_path: Path,
+ file: str,
+ selector: str,
+ mode: str,
+ bind_overrides: dict[str, str | int],
+ tamper: bool,
+) -> None:
+ path, materialized = authorized_file
+ if tamper:
+ path.write_bytes(b"x" * materialized.size)
+ raw_path = tmp_path / "guessed.pdf"
+ raw_path.write_bytes(b"not authorized")
+ target = {"resume": str(path), "raw-path": str(raw_path)}.get(file, file)
+ raw_page = FakeChooserPage()
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized, **bind_overrides)
+
+ with pytest.raises(AuthorizedFileAccessError) as refusal:
+ await attach(recording_page, target, selector, mode=mode)
+
+ assert "retained authorized bytes" not in str(refusal.value)
+ assert raw_page.clicks == []
+ assert raw_page.chosen == [] and raw_page.input_files == []
+ assert raw_page.live_listeners == 0
+ assert _recorded_types(recording_page) == []
+
+
+async def test_a_chooser_attach_on_another_page_is_refused_before_any_click(authorized_file) -> None:
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage()
+ _, attach = _bind_chooser_attach(raw_page, path, materialized)
+
+ with pytest.raises(AuthorizedFileAccessError, match="current CodeBlock page"):
+ await attach(RecordingPage(FakeChooserPage()), str(path), "#choose-file", mode="file_chooser") # type: ignore[arg-type]
+
+ assert raw_page.clicks == []
+
+
+async def test_a_chooser_attach_clears_an_agent_upload_listener_left_on_its_page(authorized_file) -> None:
+ """An agent UPLOAD_FILE whose click opened no chooser leaves its listener armed; it must not fill this chooser."""
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage()
+ foreign: list[FileChooserHandle] = []
+ foreign_handler = foreign.append
+ raw_page.on("filechooser", foreign_handler)
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)
+
+ with skyvern_context.scoped(SkyvernContext()) as context:
+ context.pending_file_chooser = PendingFileChooserListener(
+ page=raw_page, # type: ignore[arg-type]
+ file_paths=["/tmp/agent-upload.pdf"],
+ handler=foreign_handler,
+ )
+ receipt = await attach(recording_page, str(path), "#choose-file", mode="file_chooser")
+ assert context.pending_file_chooser is None
+
+ assert foreign == []
+ assert receipt == {"filename": "resume.pdf", "size": len(b"retained authorized bytes")}
+ assert [files["buffer"] for files in raw_page.chosen] == [b"retained authorized bytes"]
+ assert raw_page.live_listeners == 0
+
+
+ ("remaining", "phase", "trigger_state", "dispatched"),
+ [(1.4, "chooser", "confirmed", 1), (0.5, "trigger", "not_attempted", 0)],
+ ids=["bound-clamped-to-the-deadline", "no-time-left-to-click"],
+)
+async def test_a_chooser_attach_returns_its_typed_failure_before_the_block_deadline(
+ authorized_file, remaining: float, phase: str, trigger_state: str, dispatched: int
+) -> None:
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage(opens_chooser=False)
+ deadline = time.monotonic() + remaining
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized, deadline=deadline)
+
+ with pytest.raises(AuthorizedFileChooserError) as failure:
+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")
+
+ assert time.monotonic() < deadline
+ assert (failure.value.phase, failure.value.trigger_state) == (phase, trigger_state)
+ assert raw_page.dispatched_clicks == dispatched
+ assert raw_page.live_listeners == 0
+
+
+def test_only_a_playwright_family_page_offers_chooser_mode() -> None:
+ assert authorized_file_attach_modes(FakeChooserPage()) == {"input", "file_chooser"}
+ assert authorized_file_attach_modes(FakeRawCDPChooserPage()) == {"input"}
+ assert authorized_file_attach_modes(_RawPage()) == {"input"}
+ assert pin_file_chooser(FakeRawCDPChooserPage()) is None
+
+
+async def test_a_page_without_chooser_mode_reports_it_unavailable_instead_of_filling_an_input(authorized_file) -> None:
+ path, materialized = authorized_file
+ raw_page = FakeRawCDPChooserPage()
+ recording_page, attach = _bind_chooser_attach(raw_page, path, materialized)
+
+ with pytest.raises(AuthorizedFileChooserError) as failure:
+ await attach(recording_page, str(path), "#choose-file", mode="file_chooser")
+
+ assert str(failure.value) == AUTHORIZED_FILE_CHOOSER_UNAVAILABLE_ERROR
+ assert (failure.value.phase, failure.value.trigger_state) == ("chooser", "not_attempted")
+ assert raw_page.clicks == [] and raw_page.input_files == []
+ assert await attach(recording_page, str(path), "#direct-file") == {
+ "filename": "resume.pdf",
+ "size": len(b"retained authorized bytes"),
+ }
+ assert len(raw_page.input_files) == 1
+
+
+async def test_an_inline_chooser_failure_after_the_click_is_not_healable(authorized_file) -> None:
+ """The click may have fired, so self-heal must not re-run the block and click the trigger a second time."""
+ path, materialized = authorized_file
+ raw_page = FakeChooserPage(click_error=PlaywrightError("element was detached"))
+ recording_page = RecordingPage(raw_page) # type: ignore[arg-type]
+ now = datetime.now(UTC)
+ block = CodeBlock(
+ label="chooser_block",
+ code='await attach_authorized_file(page, resume, "#choose-file", mode="file_chooser")',
+ output_parameter=OutputParameter(
+ parameter_type=ParameterType.OUTPUT,
+ key="chooser_output",
+ description="test output",
+ output_parameter_id="op_chooser",
+ workflow_id="w_test",
+ created_at=now,
+ modified_at=now,
+ ),
+ )
+ run = block.generate_async_user_function(
+ block.code,
+ recording_page, # type: ignore[arg-type]
+ {"resume": str(path)},
+ workflow_run_id="wr_authorized",
+ organization_id="org_authorized",
+ authorized_file_materializations={"resume": materialized},
+ )
+
+ with pytest.raises(AuthorizedFileChooserError) as failure:
+ await run()
+
+ assert (failure.value.phase, failure.value.trigger_state) == ("trigger", "unknown")
+ assert block._is_healable_page_failure(failure.value, recording_page) is False
+ assert raw_page.dispatched_clicks == 1
+ assert _recorded_types(recording_page) == [ActionType.CLICK]
diff --git a/tests/unit/test_code_block_sandbox.py b/tests/unit/test_code_block_sandbox.py
index 7ed20ad00..89ffee812 100644
--- a/tests/unit/test_code_block_sandbox.py
+++ b/tests/unit/test_code_block_sandbox.py
@@ -3941,6 +3941,67 @@ class TestOpenPageHelperBinding:
assert result.success is False
assert "Failed on opened page: https://example.test/detail/3" in result.failure_reason
+ @pytest.mark.asyncio
+ async def test_a_failure_on_a_claimed_popup_names_that_popup_and_leaves_it_open(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ context = FakeSearchBrowserContext()
+ timeout = PlaywrightTimeoutError("locator timed out")
+
+ class PopupLocator:
+ def __init__(self, page: FakeSearchPage) -> None:
+ self.page = page
+
+ async def click(self, **_kwargs: Any) -> None:
+ raise timeout
+
+ async def evaluate(self, _script: str, **_kwargs: Any) -> str:
+ return '<div id="cover">'
+
+ class PopupPage(FakeSearchPage):
+ def locator(self, _selector: str, **_kwargs: Any) -> PopupLocator:
+ return PopupLocator(self)
+
+ popup = PopupPage(context=context)
+ popup.url = "https://example.test/sign-in"
+
+ @asynccontextmanager
+ async def expect_popup(**_kwargs: Any) -> AsyncIterator[SimpleNamespace]:
+ value: asyncio.Future[PopupPage] = asyncio.get_running_loop().create_future()
+ value.set_result(popup)
+ yield SimpleNamespace(value=value)
+
+ root = SimpleNamespace(
+ context=context, url="https://example.test/listing", is_closed=lambda: False, expect_popup=expect_popup
+ )
+ root.title = AsyncMock(return_value="Listing")
+ browser_state = SimpleNamespace(
+ browser_artifacts=BrowserArtifacts(),
+ get_working_page=AsyncMock(return_value=root),
+ list_valid_pages=AsyncMock(return_value=[root]),
+ set_active_page=AsyncMock(),
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.workflow.models.block.app.AGENT_FUNCTION.validate_code_block", AsyncMock()
+ )
+ monkeypatch.setattr(CodeBlock, "get_or_create_browser_state", AsyncMock(return_value=browser_state))
+ monkeypatch.setattr(CodeBlock, "get_workflow_run_context", lambda *args: FakeWorkflowRunContext(values={}))
+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", AsyncMock())
+ block = self._block()
+ block.code = (
+ "async with page.expect_popup() as popup_info:\n"
+ " pass\n"
+ "popup = await popup_info.value\n"
+ "await popup.locator('#user').click()\n"
+ )
+
+ result = await block.execute(workflow_run_id="wrid_test", workflow_run_block_id="")
+
+ assert result.success is False
+ assert "Final URL: https://example.test/listing" in result.failure_reason
+ assert "Failed on opened page: https://example.test/sign-in" in result.failure_reason
+ assert popup.closed is False
+
@pytest.mark.asyncio
async def test_open_page_fails_closed_without_a_run_browser(self) -> None:
with pytest.raises(RuntimeError, match="only supported while the run browser is open"):
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 715e000a9..8e31f721e 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -108,6 +108,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
)
from skyvern.forge.sdk.copilot.enforcement import (
NUDGE_SENTINEL,
+ RAW_SECRET_REPLY_WITHHELD_OBSERVATION,
CopilotNonRetriableNavError,
CopilotTotalTimeoutError,
CopilotUnrecoverableToolError,
@@ -127,6 +128,7 @@ from skyvern.forge.sdk.copilot.output_policy import OutputPolicyReason, OutputPo
from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure
from skyvern.forge.sdk.copilot.request_policy import (
_REDACTED_REFUSED_SECRET_TURN,
+ RAW_SECRET_REFUSAL_SENTINEL,
TRANSCRIPT_ANCHOR_CHAR_CAP,
CompletionCriterion,
RequestPolicy,
@@ -7476,6 +7478,58 @@ class TestCopilotConfig:
)
assert blocked.result is blocked_result
+ @staticmethod
+ def _withheld_reply(*reason_codes: OutputPolicyReason) -> OutputGuardrailTripwireTriggered:
+ guardrail_result = MagicMock()
+ guardrail_result.output.output_info = {"allowed": False, "reason_codes": [code.value for code in reason_codes]}
+ return OutputGuardrailTripwireTriggered(guardrail_result)
+
+ @pytest.mark.asyncio
+ async def test_reply_withheld_for_a_raw_secret_shape_gets_one_tool_less_reask(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ run = await _run_tool_bearing_empty_completion_turn(
+ monkeypatch,
+ _fake_run_result({"type": "REPLY", "user_response": "The page shows receipt FV-0F9080EE."}),
+ attempts=(_EnforcementAttempt(error=self._withheld_reply(OutputPolicyReason.RAW_SECRET_LEAK)),),
+ )
+
+ [call] = run.drain_calls
+ assert call.current_input == RAW_SECRET_REPLY_WITHHELD_OBSERVATION
+ assert call.model_settings.tool_choice == "none"
+ assert call.max_turns == 1
+ assert run.result.user_response == "The page shows receipt FV-0F9080EE."
+
+ @pytest.mark.asyncio
+ async def test_reask_that_still_carries_a_raw_secret_keeps_the_refusal(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ run = await _run_tool_bearing_empty_completion_turn(
+ monkeypatch,
+ self._withheld_reply(OutputPolicyReason.RAW_SECRET_LEAK),
+ attempts=(_EnforcementAttempt(error=self._withheld_reply(OutputPolicyReason.RAW_SECRET_LEAK)),),
+ )
+
+ assert len(run.drain_calls) == 1
+ assert RAW_SECRET_REFUSAL_SENTINEL in run.result.user_response
+
+ @pytest.mark.asyncio
+ async def test_reply_withheld_for_another_reason_is_not_reasked(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ run = await _run_tool_bearing_empty_completion_turn(
+ monkeypatch,
+ _fake_run_result({"type": "REPLY", "user_response": "unused"}),
+ attempts=(
+ _EnforcementAttempt(
+ error=self._withheld_reply(
+ OutputPolicyReason.RAW_SECRET_LEAK, OutputPolicyReason.PERSISTENCE_STATE_MISMATCH
+ )
+ ),
+ ),
+ )
+
+ assert run.drain_calls == []
+ assert run.result.user_response != "unused"
+
@pytest.mark.asyncio
async def test_tool_bearing_empty_completion_final_reply_is_not_retried_for_untested_draft(
self, monkeypatch: pytest.MonkeyPatch
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index d2b57225e..a850d1ad5 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -87,6 +87,7 @@ from skyvern.forge.sdk.copilot.secret_scrub import clear_session_scrub_values, r
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.tools.composition_capture import store_post_run_page_evidence
from skyvern.forge.sdk.copilot.tools.run_execution import (
+ COPILOT_PROPOSAL_CHANGED_ERROR,
_authored_literal_locator_selectors,
_carry_unresolved_failure_into_result,
_failed_block_code,
@@ -122,6 +123,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (
Workflow,
WorkflowDefinition,
WorkflowRunStatus,
+ workflow_definition_sha256,
)
from skyvern.schemas.self_heal import HealEpisode, HealSkipReason, HealStatus
from skyvern.schemas.workflows import BlockType
@@ -136,8 +138,10 @@ from tests.unit.copilot_test_helpers import (
count_record_and_send,
failed_second_factor_run,
handback_ctx,
+ historical_code_version,
inert_approval_workflow,
install_get_run_results_harness,
+ install_historical_run,
install_run_blocks_harness,
make_copilot_ctx,
make_stub_html_artifact,
@@ -2061,6 +2065,8 @@ async def test_failed_run_complete_fact_packet_reaches_ordinary_repair_input(
)
run_workflow = SimpleNamespace(
workflow_id="wf_run_snapshot",
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ version=1,
created_by=None,
modified_at=datetime(2026, 4, 21, 12, 0),
organization_id=ctx.organization_id,
@@ -2077,12 +2083,14 @@ async def test_failed_run_complete_fact_packet_reaches_ordinary_repair_input(
)
run = SimpleNamespace(
workflow_run_id="wr_failed_complete_packet",
+ workflow_id="wf_run_snapshot",
workflow_permanent_id=ctx.workflow_permanent_id,
browser_session_id="pbs_failed_complete_packet",
status="failed",
failure_reason="Code block failed.",
created_at=datetime(2026, 4, 21, 12, 0),
trigger_type=None,
+ workflow_definition_sha256=None,
)
block = SimpleNamespace(
workflow_run_block_id="wrb_failed_complete_packet",
@@ -2209,7 +2217,8 @@ async def test_failed_run_complete_fact_packet_reaches_ordinary_repair_input(
workflow_run_id="wr_failed_complete_packet",
)
assert nonassociated is not None
- assert "execution_source" not in nonassociated["run"]
+ assert nonassociated["run"]["execution_source"]["source_kind"] == "run_version"
+ assert nonassociated["run"]["execution_source"]["workflow_id"] == "wf_run_snapshot"
def test_unavailable_registered_output_rows_do_not_become_false_missing_output_facts() -> None:
@@ -6048,7 +6057,7 @@ async def test_candidate_run_bind_uses_the_call_owned_source_and_token(
assert result == {
"ok": False,
- "error": "The pending Copilot proposal changed before the test started; reload and try again.",
+ "error": COPILOT_PROPOSAL_CHANGED_ERROR,
}
assert captured["expected_owner_turn_id"] == "turn-source"
assert captured["expected_revision"] == 3
@@ -8720,9 +8729,9 @@ async def test_read_run_reports_the_executed_versions_loop_inputs_not_the_curren
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
- executed = (
- await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="w_source")
- ).model_copy(update={"modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)})
+ executed = (await inert_approval_workflow(_parse_then_loop_yaml("current_value"), workflow_id="wf-1")).model_copy(
+ update={"workflow_permanent_id": "wpid-1", "modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)}
+ )
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
@@ -8738,7 +8747,7 @@ async def test_read_run_reports_the_executed_versions_loop_inputs_not_the_curren
page = sanitize_tool_result_for_llm("get_run_results", project_run_results_page(result, {}))["data"]
assert page["loop_inputs"]["workflow_run_id"] == "wr-1"
- assert page["loop_inputs"]["workflow_id"] == "w_source"
+ assert page["loop_inputs"]["workflow_id"] == "wf-1"
[loop] = page["loop_inputs"]["loops"]
assert (loop["loop_variable_reference"], loop["selected_input"], loop["loop_values_counts"]) == (
"current_value",
@@ -8773,8 +8782,12 @@ async def test_read_run_withholds_loop_inputs_when_the_workflow_row_changed_afte
monkeypatch: pytest.MonkeyPatch, workflow_yaml: str, created_by: str | None, expected_unavailable: bool
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
- executed = (await inert_approval_workflow(workflow_yaml, workflow_id="w_source")).model_copy(
- update={"created_by": created_by, "modified_at": HARNESS_RUN_CREATED_AT + timedelta(minutes=5)}
+ executed = (await inert_approval_workflow(workflow_yaml, workflow_id="wf-1")).model_copy(
+ update={
+ "workflow_permanent_id": "wpid-1",
+ "created_by": created_by,
+ "modified_at": HARNESS_RUN_CREATED_AT + timedelta(minutes=5),
+ }
)
ctx = install_get_run_results_harness(
monkeypatch,
@@ -8874,8 +8887,10 @@ async def test_prior_run_packet_and_handoff_carry_the_notice_for_a_truncated_loo
) -> None:
monkeypatch.setattr(forge_app.WORKFLOW_SERVICE, "get_workflow_by_permanent_id", AsyncMock(return_value=None))
executed = (
- await inert_approval_workflow(_parse_then_loop_yaml("current_value." + "a" * 400), workflow_id="w_source")
- ).model_copy(update={"modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)})
+ await inert_approval_workflow(_parse_then_loop_yaml("current_value." + "a" * 400), workflow_id="wf-1")
+ ).model_copy(
+ update={"workflow_permanent_id": "wpid-1", "modified_at": HARNESS_RUN_CREATED_AT - timedelta(minutes=1)}
+ )
ctx = install_get_run_results_harness(
monkeypatch,
blocks=[
@@ -8949,3 +8964,187 @@ def test_many_long_loop_facts_stay_inside_the_packet_budget(ai_fallback_rows: in
assert all(len(loop.loop_variable_reference or "") <= 200 for loop in projected.run.loop_inputs.loops)
assert all(loop.selected_input == "loop_variable_reference" for loop in projected.run.loop_inputs.loops)
assert all(loop.loop_values_counts == [1_000_000] * 10 for loop in projected.run.loop_inputs.loops)
+
+
+_EXECUTED_A_CODE = 'greeting = "Hello " + legacy_applicant_name\nreturn {"greeting": greeting}\n'
+_EDITABLE_B_CODE = 'display = greeting_prefix + " " + applicant_display_name\nreturn {"greeting": display}\n'
+_EDITABLE_B_YAML = """title: Historical run source
+workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ key: full_name
+ workflow_parameter_type: string
+ - parameter_type: workflow
+ key: greeting_prefix
+ workflow_parameter_type: string
+ blocks:
+ - block_type: code
+ label: greet_applicant
+ parameter_keys: [full_name, greeting_prefix]
+ code: |
+ display = greeting_prefix + " " + applicant_display_name
+ return {"greeting": display}
+"""
+
+
+def _editable_b_ctx() -> CopilotContext:
+ return make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml=_EDITABLE_B_YAML)
+
+
+async def test_get_run_results_and_hydration_name_the_version_a_finished_run_executed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ install_historical_run(
+ monkeypatch, historical_code_version({"greet_applicant": _EXECUTED_A_CODE}), failed_label="greet_applicant"
+ )
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args: None)
+ ctx = _editable_b_ctx()
+
+ raw = await tools_module.get_run_results_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="get_run_results"), json.dumps({"workflow_run_id": "wr-1"})
+ )
+ hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")
+
+ tool_source = json.loads(raw)["data"]["execution_source"]
+ assert hydrated is not None
+ assert hydrated["run"]["execution_source"] == tool_source
+ assert tool_source == {
+ "source_kind": "run_version",
+ "disposition": "available",
+ "workflow_run_id": "wr-1",
+ "workflow_id": "wf-1",
+ "workflow_version": 3,
+ "failed_block_label": "greet_applicant",
+ "failed_statement": 'greeting = "Hello " + legacy_applicant_name',
+ "declared_parameter_keys": ["applicant_name"],
+ "block_code_sha256": {"greet_applicant": hashlib.sha256(_EXECUTED_A_CODE.encode()).hexdigest()},
+ "current_draft_code_matches": {"greet_applicant": False},
+ "failed_block_code": _EXECUTED_A_CODE,
+ }
+ executed_fields = json.dumps(tool_source)
+ assert "applicant_display_name" not in executed_fields
+ assert hashlib.sha256(_EDITABLE_B_CODE.encode()).hexdigest() not in executed_fields
+
+
+async def test_a_historical_read_takes_no_signature_or_unverified_label_from_the_current_draft(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ install_historical_run(
+ monkeypatch, historical_code_version({"greet_applicant": _EXECUTED_A_CODE}), failed_label="greet_applicant"
+ )
+ ctx = _editable_b_ctx()
+ ctx.last_unverified_block_labels = ["greet_applicant", "only_in_b"]
+ ctx.unbound_required_parameter_keys = ["full_name"]
+ ctx.runner_code_block_associations_by_label = {"greet_applicant": "assoc_b"}
+ history_before = copy.deepcopy(ctx.recorded_build_test_outcome_history)
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+ outcome = run_execution_module._build_recorded_build_test_outcome(ctx, result, recorded_run_outcome=None)
+ hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")
+ draft_outcome = run_execution_module._build_recorded_build_test_outcome(
+ ctx,
+ {**result, "data": {key: value for key, value in result["data"].items() if key != "execution_source"}},
+ recorded_run_outcome=None,
+ )
+
+ assert draft_outcome is not None and draft_outcome.authored_structure_signature
+ assert outcome is not None
+ assert outcome.workflow_run_id == "wr-1"
+ assert outcome.authored_structure_signature is None
+ assert outcome.block_shape_hashes == {}
+ assert outcome.executed_block_associations == ()
+ assert hydrated is not None
+ assert [item for item in hydrated.get("unfinished_items", []) if item.get("kind") == "unverified_block"] == []
+ assert ctx.recorded_build_test_outcome_history == history_before
+ assert ctx.latest_recorded_build_test_outcome is None
+
+
[email protected]("definition_matches", [True, False], ids=["metadata-only-edit", "definition-rewritten"])
+async def test_the_definition_recorded_at_dispatch_decides_whether_an_edited_row_is_what_ran(
+ monkeypatch: pytest.MonkeyPatch, definition_matches: bool
+) -> None:
+ # A folder or title edit bumps modified_at past the run; only a definition change may withhold its source.
+ version = historical_code_version(
+ {"greet_applicant": _EXECUTED_A_CODE}, modified_at=HARNESS_RUN_CREATED_AT + timedelta(minutes=5)
+ )
+ recorded = historical_code_version({"greet_applicant": _EXECUTED_A_CODE if definition_matches else "pass\n"})
+ install_historical_run(
+ monkeypatch,
+ version,
+ failed_label="greet_applicant",
+ run_definition_sha256=workflow_definition_sha256(recorded.workflow_definition),
+ )
+ observe_locators = AsyncMock(return_value=None)
+ monkeypatch.setattr(run_execution_module, "_observe_authored_locators", observe_locators)
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, _editable_b_ctx())
+
+ source = result["data"]["execution_source"]
+ if definition_matches:
+ assert source["disposition"] == "available"
+ assert source["failed_statement"] == 'greeting = "Hello " + legacy_applicant_name'
+ assert observe_locators.await_args.kwargs["failed_block_code"] == _EXECUTED_A_CODE
+ assert "requested_output_parameter_definitions" in result["data"]
+ else:
+ assert source == {
+ "source_kind": "run_version",
+ "disposition": "unavailable",
+ "workflow_run_id": "wr-1",
+ "workflow_id": "wf-1",
+ "reason": "overwritten_after_run_start",
+ }
+ assert "requested_output_parameter_definitions" not in result["data"]
+
+
+ ("version", "reason"),
+ [
+ (None, "version_missing"),
+ (historical_code_version({"greet_applicant": _EXECUTED_A_CODE}, workflow_id="wf-other"), "identity_mismatch"),
+ (
+ historical_code_version(
+ {"greet_applicant": _EXECUTED_A_CODE}, modified_at=HARNESS_RUN_CREATED_AT + timedelta(seconds=1)
+ ),
+ "overwritten_after_run_start",
+ ),
+ (historical_code_version({"greet_applicant": _EXECUTED_A_CODE}), None),
+ ],
+)
+async def test_an_unprovable_executed_version_is_unavailable_and_never_falls_back_to_the_draft(
+ monkeypatch: pytest.MonkeyPatch, version: SimpleNamespace | None, reason: str | None
+) -> None:
+ install_historical_run(monkeypatch, version, failed_label="greet_applicant")
+ observe_locators = AsyncMock(return_value=None)
+ monkeypatch.setattr(run_execution_module, "_observe_authored_locators", observe_locators)
+ ctx = _editable_b_ctx()
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+ outcome = run_execution_module._build_recorded_build_test_outcome(ctx, result, recorded_run_outcome=None)
+
+ source = result["data"]["execution_source"]
+ observed_code = observe_locators.await_args.kwargs["failed_block_code"] if observe_locators.await_args else None
+ assert outcome is not None and outcome.authored_structure_signature is None
+ assert "applicant_display_name" not in json.dumps(source)
+ fetched_terminal_page = run_execution_module._fetch_dispatched_terminal_page_evidence.await_count > 0
+ if reason is None:
+ assert source["disposition"] == "available"
+ assert source["failed_statement"] == 'greeting = "Hello " + legacy_applicant_name'
+ assert observed_code == _EXECUTED_A_CODE
+ assert "requested_output_parameter_definitions" in result["data"]
+ assert fetched_terminal_page
+ return
+ # An unproven row may have lost the sensitive parameters or outputs the run had, so nothing is read from it.
+ assert observed_code is None
+ assert "requested_output_parameter_definitions" not in result["data"]
+ assert not fetched_terminal_page
+ assert source == {
+ "source_kind": "run_version",
+ "disposition": "unavailable",
+ "workflow_run_id": "wr-1",
+ "workflow_id": "wf-1",
+ "reason": reason,
+ }
diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.py
index 4056f00c9..5d2b76a0b 100644
--- a/tests/unit/test_copilot_credential_pause.py
+++ b/tests/unit/test_copilot_credential_pause.py
@@ -20,9 +20,11 @@ Covers:
from __future__ import annotations
import asyncio
+import io
import json
+import logging
import time
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Iterator
from copy import deepcopy
from dataclasses import replace
from datetime import datetime, timedelta, timezone
@@ -32,12 +34,14 @@ from typing import Any, Literal
from unittest.mock import AsyncMock, MagicMock
import pytest
+import structlog
from agents import RunConfig
from agents.tool_context import ToolContext
-from fastapi import HTTPException, status
+from fastapi import BackgroundTasks, HTTPException, status
from structlog.testing import capture_logs
from skyvern.config import settings
+from skyvern.exceptions import HttpException as VaultHttpException
from skyvern.forge import app
from skyvern.forge.sdk.cache.base import NoopLock
from skyvern.forge.sdk.copilot import credential_pause as credential_pause_module
@@ -56,11 +60,14 @@ from skyvern.forge.sdk.copilot.context import (
StructuredContext,
record_approved_credentials_in_global_llm_context,
)
+from skyvern.forge.sdk.copilot.credential_generation import SAFE_SYMBOLS, generate_registration_password
from skyvern.forge.sdk.copilot.credential_pause import (
CredentialPauseRejection,
CredentialPauseResolution,
+ _decode_active_pause,
_encode_active_pause,
await_pending_credential_pause,
+ claim_credential_generation,
credential_pause_active_key,
credential_response_cache_key,
encode_credential_response,
@@ -96,20 +103,30 @@ from skyvern.forge.sdk.copilot.tools._shared import TOTAL_TIMEOUT_SECONDS, _copi
from skyvern.forge.sdk.copilot.tools.credential_fill import _request_credential
from skyvern.forge.sdk.copilot.tools.guardrails import _authority_tool_error
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
+from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.event_source_stream import EventSourceStream
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.forge_log import setup_logger
+from skyvern.forge.sdk.routes import workflow_copilot as copilot_routes
from skyvern.forge.sdk.routes.workflow_copilot import (
WorkflowCopilotCredentialResponseRequest,
+ workflow_copilot_credential_generate,
workflow_copilot_credential_response,
)
from skyvern.forge.sdk.schemas.credentials import (
+ CreateCredentialRequest,
Credential,
+ CredentialItem,
CredentialType,
CredentialVaultType,
PasswordCredential,
TotpType,
)
from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotCredentialGenerateRequest,
+ WorkflowCopilotCredentialGenerateResult,
WorkflowCopilotCredentialPauseResolvedUpdate,
+ WorkflowCopilotCredentialRegistration,
WorkflowCopilotCredentialRequiredUpdate,
WorkflowCopilotStreamMessageType,
)
@@ -1718,6 +1735,22 @@ def test_connected_resume_stamps_nothing_when_the_ask_spans_several_origins() ->
assert policy.live_page_admitted_urls == {}
+async def test_generated_credential_connected_from_a_multi_origin_ask_gets_no_fill_grant(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _tool_ctx(monkeypatch)
+ ctx.request_policy = _ask_origin_policy("https://portal.example.com/login", "https://other.example.com/login")
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.org_credentials_for_turn = []
+
+ credential_pause_module._apply_connected_credential_to_policy(ctx, ctx.request_policy, _make_credential("cred_new"))
+ grant, error = await tools_module.credential_fill._credential_fill_origin_grant(ctx, "cred_new")
+
+ assert ctx.request_policy.live_page_admitted_urls == {}
+ assert grant is None and error
+
+
def test_connected_resume_names_the_picked_credential_superseding_an_earlier_mention() -> None:
"""The fill seam's which-credential check is set equality, so a card answer must replace an
earlier prose mention, not join it."""
@@ -2894,3 +2927,913 @@ async def test_a_run_derived_card_answered_by_signing_in_resumes_with_facts_and_
assert ctx.credential_pause_outcome == "signed_in"
assert credential_prompt_reason(ctx.request_policy, None) is None
assert await maybe_credential_pause(ctx, _fake_result(), _make_stream(), config) is None
+
+
+_REGISTRATION = {
+ "username": "[email protected]",
+ "credential_name": "Portal test account",
+ "password_length": 24,
+ "charset": "alphanumeric_symbols",
+}
+
+
+ ("length", "charset", "classes"),
+ [
+ (24, "alphanumeric_symbols", 4),
+ (128, "alphanumeric_symbols", 4),
+ (24, "alphanumeric", 3),
+ (128, "alphanumeric", 3),
+ ],
+)
+def test_registration_password_has_the_length_and_every_class_of_its_charset(
+ length: int, charset: Literal["alphanumeric", "alphanumeric_symbols"], classes: int
+) -> None:
+ for _ in range(50):
+ password = generate_registration_password(length, charset)
+ present = [
+ any(c.islower() for c in password),
+ any(c.isupper() for c in password),
+ any(c.isdigit() for c in password),
+ any(c in SAFE_SYMBOLS for c in password),
+ ]
+ assert len(password) == length
+ assert sum(present) == classes
+ assert all(c.isalnum() or c in SAFE_SYMBOLS for c in password)
+
+
[email protected]("length", [23, 129])
+def test_registration_password_refuses_a_length_outside_its_bounds(length: int) -> None:
+ with pytest.raises(ValueError):
+ generate_registration_password(length, "alphanumeric")
+
+
+class _RegistrationVault:
+ """Stands in for the shared create path: records what it was asked to store and can fail or stall."""
+
+ def __init__(
+ self,
+ credential: Credential,
+ *,
+ reject: bool = False,
+ fail: bool = False,
+ delay: float = 0.0,
+ stored_password: str | None = None,
+ read_delay: float = 0.0,
+ read_fails: bool = False,
+ vault_status: int | None = None,
+ prepare_delay: float = 0.0,
+ ):
+ self.credential = credential
+ self.prepare_delay = prepare_delay
+ self.vault_status = vault_status
+ self.reject = reject
+ self.fail = fail
+ self.delay = delay
+ self.stored_password = stored_password
+ self.read_delay = read_delay
+ self.read_fails = read_fails
+ self.prepared = 0
+ self.passwords: list[str] = []
+ self.stored = 0
+ self.hooks_run: list[str] = []
+
+ async def prepare(self, organization_id: str, data: CreateCredentialRequest) -> _RegistrationVault:
+ self.prepared += 1
+ await asyncio.sleep(self.prepare_delay)
+ if self.reject:
+ raise HTTPException(status_code=400, detail="No credential vault is configured")
+ return self
+
+ async def store(
+ self,
+ organization_id: str,
+ data: CreateCredentialRequest,
+ created_by: str | None,
+ service: _RegistrationVault,
+ tasks: BackgroundTasks,
+ ) -> Credential:
+ self.passwords.append(data.credential.password)
+ await asyncio.sleep(self.delay)
+ if self.fail:
+ try:
+ raise RuntimeError(f"vault rejected password '{data.credential.password}'")
+ except RuntimeError:
+ logging.getLogger("skyvern.test.vault").exception("vault create failed")
+ raise HTTPException(status_code=502, detail=f"Custom credential service returned HTTP 500 for {data.name}")
+ self.stored += 1
+ tasks.add_task(self.hooks_run.append, self.credential.credential_id)
+ return self.credential
+
+ async def create_credential(
+ self, organization_id: str, data: CreateCredentialRequest, created_by: str | None
+ ) -> Credential:
+ self.passwords.append(data.credential.password)
+ assert self.vault_status is not None
+ raise VaultHttpException(self.vault_status, "https://vault.example.test/credentials")
+
+ async def get_credential_item(self, db_credential: Credential) -> CredentialItem:
+ await asyncio.sleep(self.read_delay)
+ password = self.stored_password or self.passwords[-1]
+ if self.read_fails:
+ raise RuntimeError(f"vault read returned password '{password}' in an unexpected shape")
+ return CredentialItem(
+ item_id=db_credential.item_id,
+ name=db_credential.name,
+ credential_type=CredentialType.PASSWORD,
+ credential=PasswordCredential(username="[email protected]", password=password),
+ )
+
+ def install(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(copilot_routes, "prepare_credential_create", self.prepare)
+ if self.vault_status is None:
+ monkeypatch.setattr(copilot_routes, "store_new_credential", self.store)
+
+
+def _registration_database(
+ monkeypatch: pytest.MonkeyPatch, credential: Credential, readback: list[Credential] | None = None
+) -> SimpleNamespace:
+ database = SimpleNamespace(
+ credentials=SimpleNamespace(
+ get_credentials_by_ids=AsyncMock(return_value=[credential]),
+ get_credentials=AsyncMock(return_value=readback or []),
+ ),
+ workflow_params=SimpleNamespace(record_pending_copilot_turn_credential_resume=AsyncMock()),
+ )
+ monkeypatch.setattr(credential_pause_module.app, "DATABASE", database)
+ return database
+
+
+def _seed_registration_pause(
+ cache: _FakeCache, *, expires_at: datetime | None = None, registration: bool = True
+) -> None:
+ cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")] = _encode_active_pause(
+ "tok-1",
+ expires_at or datetime.now(timezone.utc) + timedelta(minutes=5),
+ registration=WorkflowCopilotCredentialRegistration(**_REGISTRATION) if registration else None,
+ )
+
+
+async def _generate(
+ *, resume_token: str = "tok-1", organization_id: str = "org-1"
+) -> WorkflowCopilotCredentialGenerateResult:
+ with skyvern_context.scoped(SkyvernContext()):
+ return await workflow_copilot_credential_generate(
+ WorkflowCopilotCredentialGenerateRequest(
+ turn_id="turn-1", workflow_copilot_chat_id="chat-1", resume_token=resume_token
+ ),
+ organization=SimpleNamespace(organization_id=organization_id), # type: ignore[arg-type]
+ current_user_id="user-1",
+ )
+
+
+def _registration_record(cache: _FakeCache) -> WorkflowCopilotCredentialRegistration | None:
+ record = _decode_active_pause(cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")])
+ assert record is not None and record.status == "pending"
+ return record.registration
+
+
+async def test_registration_card_generates_one_credential_and_resumes_with_only_its_id(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ cache = _FakeCache()
+ ctx = _tool_ctx(monkeypatch, cache)
+ ctx.client_supports_credential_generation = True
+ credential = _make_credential("cred_new", "Portal test account")
+ database = _registration_database(monkeypatch, credential)
+ vault = _RegistrationVault(credential)
+ vault.install(monkeypatch)
+ responses: list[WorkflowCopilotCredentialGenerateResult] = []
+
+ async def answer_with_generate(
+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
+ ) -> bool:
+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
+ responses.append(await _generate(resume_token=frame.resume_token))
+ return True
+
+ ctx.stream.send.side_effect = answer_with_generate
+ with capture_logs() as logs:
+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)
+
+ [card] = _sent_cards(ctx)
+ assert card.reason == "credential_registration"
+ assert card.login_page_urls == ["https://portal.example.com/login"]
+ assert card.registration is not None
+ assert (card.registration.username, card.registration.credential_name) == (
+ "Portal test account",
+ )
+ assert [response.result for response in responses] == ["connected"]
+ assert result["status"] == "connected"
+ assert result["credential_id"] == "cred_new"
+ assert (result["generated"], result["site_effect"]) == (True, "none")
+ assert "next" not in result and "data" not in result
+ assert ctx.request_policy.live_page_admitted_urls == {"cred_new": "https://portal.example.com/login"}
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ grant, error = await tools_module.credential_fill._credential_fill_origin_grant(ctx, "cred_new")
+ assert error is None and grant is not None
+ assert tools_module.credential_fill._within_grant("https://portal.example.com/signup", grant)
+ assert not tools_module.credential_fill._within_grant("https://elsewhere.example.com/signup", grant)
+ assert vault.stored == 1
+ await asyncio.sleep(0)
+ assert vault.hooks_run == ["cred_new"]
+ database.workflow_params.record_pending_copilot_turn_credential_resume.assert_awaited()
+ [password] = vault.passwords
+ surfaces = [
+ json.dumps(result),
+ *(call.args[0].model_dump_json() for call in ctx.stream.send.await_args_list),
+ *(str(value) for value in cache.store.values()),
+ *(response.model_dump_json() for response in responses),
+ json.dumps(logs, default=str),
+ ]
+ assert all(password not in surface for surface in surfaces)
+
+
[email protected]("answer", ["generate", "pick"])
+async def test_registration_card_during_origin_recovery_admits_its_generated_credential_without_site_evidence(
+ monkeypatch: pytest.MonkeyPatch, answer: str
+) -> None:
+ cache = _FakeCache()
+ ctx = _recovery_ctx(monkeypatch, cache)
+ ctx.client_supports_credential_generation = True
+ generated = _make_credential("cred_new", "Portal test account")
+ picked = _make_credential("cred_blank")
+ database = _registration_database(monkeypatch, generated if answer == "generate" else picked)
+ database.credentials.get_credential = AsyncMock(return_value=None)
+ vault = _RegistrationVault(generated)
+ vault.install(monkeypatch)
+
+ async def answer_card(
+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
+ ) -> bool:
+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
+ if answer == "generate":
+ await _generate(resume_token=frame.resume_token)
+ else:
+ cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")] = encode_credential_response(
+ "connected", "cred_blank", frame.resume_token
+ )
+ return True
+
+ ctx.stream.send.side_effect = answer_card
+ result = await _call_ask_tool(ctx, login_page_url=f"{_IDP_ORIGIN}/login", registration=_REGISTRATION)
+
+ [card] = _sent_cards(ctx)
+ assert card.registration is not None
+ if answer == "generate":
+ assert (result["status"], result["credential_id"]) == ("connected", "cred_new")
+ assert ctx.credential_pause_outcome == "connected"
+ assert ctx.request_policy.live_page_admitted_urls["cred_new"] == f"{_IDP_ORIGIN}/login"
+ assert ctx.credential_origin_recovery is None
+ else:
+ assert vault.passwords == []
+ assert "cred_blank" not in ctx.request_policy.live_page_admitted_urls
+ assert ctx.credential_pause_outcome == "not_admitted"
+ _assert_declined(ctx, result, "connected_other_site")
+
+
+async def test_registration_ask_on_an_origin_recovery_handback_after_a_generate_raises_no_card(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ cache = _FakeCache()
+ ctx = _tool_ctx(monkeypatch, cache)
+ ctx.client_supports_credential_generation = True
+ generated = _make_credential("cred_new", "Portal test account")
+ _registration_database(monkeypatch, generated)
+ vault = _RegistrationVault(generated)
+ vault.install(monkeypatch)
+
+ async def answer_with_generate(
+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
+ ) -> bool:
+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
+ await _generate(resume_token=frame.resume_token)
+ return True
+
+ ctx.stream.send.side_effect = answer_with_generate
+ first = await _call_ask_tool(ctx, registration=_REGISTRATION)
+ assert first["status"] == "connected"
+ ctx.credential_origin_recovery = _PENDING
+
+ handback = await _call_ask_tool(ctx, login_page_url=f"{_IDP_ORIGIN}/login", registration=_REGISTRATION)
+
+ assert handback["status"] == "already_generated"
+ assert len(_sent_cards(ctx)) == 1
+ assert (vault.stored, len(vault.passwords)) == (1, 1)
+ assert ctx.credential_origin_recovery == _PENDING
+
+
[email protected]("reask", ["origin_recovery_handback", "run_budget_reset"])
[email protected]("outcome", ["unknown", "rejected"])
+async def test_registration_reask_after_an_unconfirmed_generate_never_creates_a_second_credential(
+ monkeypatch: pytest.MonkeyPatch, outcome: str, reask: str
+) -> None:
+ cache = _FakeCache()
+ ctx = _tool_ctx(monkeypatch, cache)
+ ctx.client_supports_credential_generation = True
+ _registration_database(monkeypatch, _make_credential("cred_new", "Portal test account"))
+ vault = _RegistrationVault(
+ _make_credential("cred_new", "Portal test account"), reject=outcome == "rejected", fail=outcome == "unknown"
+ )
+ vault.install(monkeypatch)
+ generated: list[str] = []
+
+ async def generate_then_skip(
+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
+ ) -> bool:
+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
+ generated.append((await _generate(resume_token=frame.resume_token)).result)
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id=frame.turn_id,
+ resume_token=frame.resume_token,
+ action="skip",
+ credential_id=None,
+ )
+ return True
+
+ ctx.stream.send.side_effect = generate_then_skip
+ first = await _call_ask_tool(ctx, registration=_REGISTRATION)
+ assert (first["status"], generated) == ("skipped", [outcome])
+ assert ctx.credential_pause_reaskable_by_run is True
+ login_page_url = "https://portal.example.com/login"
+ if reask == "origin_recovery_handback":
+ ctx.credential_origin_recovery = _PENDING
+ login_page_url = f"{_IDP_ORIGIN}/login"
+ else:
+ ctx.credential_pause_used = False
+
+ second = await _call_ask_tool(ctx, login_page_url=login_page_url, registration=_REGISTRATION)
+
+ if outcome == "unknown":
+ assert second["status"] == "already_generated"
+ assert len(_sent_cards(ctx)) == 1
+ assert (vault.prepared, len(vault.passwords), generated) == (1, 1, ["unknown"])
+ else:
+ assert len(_sent_cards(ctx)) == 2
+ assert (vault.prepared, vault.passwords, generated) == (2, [], ["rejected", "rejected"])
+
+
+async def test_registration_card_skipped_creates_no_credential_and_cannot_generate_afterwards(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ cache = _FakeCache()
+ ctx = _tool_ctx(monkeypatch, cache)
+ ctx.client_supports_credential_generation = True
+ _registration_database(monkeypatch, _make_credential())
+ vault = _RegistrationVault(_make_credential())
+ vault.install(monkeypatch)
+ ctx.stream.send.side_effect = _answer_each_card(cache, [("skip", None)])
+
+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)
+
+ assert result["status"] == "skipped"
+ with pytest.raises(HTTPException) as late:
+ await _generate()
+ assert late.value.status_code == status.HTTP_409_CONFLICT
+ assert (vault.prepared, vault.passwords) == (0, [])
+
+
+async def test_registration_card_answered_with_a_saved_login_does_not_report_it_generated(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ cache = _FakeCache()
+ ctx = _tool_ctx(monkeypatch, cache)
+ ctx.client_supports_credential_generation = True
+ _registration_database(monkeypatch, _make_credential("cred_saved", "Saved login"))
+ vault = _RegistrationVault(_make_credential())
+ vault.install(monkeypatch)
+ ctx.stream.send.side_effect = _answer_each_card(cache, [("connected", "cred_saved")])
+
+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)
+
+ assert (result["status"], result["credential_id"]) == ("connected", "cred_saved")
+ assert "generated" not in result and "site_effect" not in result
+ assert (vault.prepared, vault.passwords) == (0, [])
+
+
+ ("registration", "expected"),
+ [
+ ({**_REGISTRATION, "password_length": 23}, "unsupported_constraints"),
+ ({**_REGISTRATION, "password_length": 129}, "unsupported_constraints"),
+ ({**_REGISTRATION, "username": " "}, None),
+ ],
+)
+async def test_registration_ask_outside_the_typed_bounds_raises_no_card(
+ monkeypatch: pytest.MonkeyPatch, registration: dict[str, Any], expected: str | None
+) -> None:
+ ctx = _tool_ctx(monkeypatch)
+ ctx.client_supports_credential_generation = True
+
+ result = await _call_ask_tool(ctx, registration=registration)
+
+ assert result.get("status") == expected
+ assert "fallback" not in result and "next" not in result
+ assert _sent_cards(ctx) == []
+ assert ctx.credential_pause_used is False
+
+
+async def test_registration_ask_is_unavailable_when_the_client_cannot_generate(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _tool_ctx(monkeypatch)
+ ctx.client_supports_credential_generation = False
+
+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)
+
+ assert result["status"] == "unavailable"
+ assert "fallback" not in result and "next" not in result
+ assert _sent_cards(ctx) == []
+
+
[email protected]("conflict", ["credential_id", "raw_secret"])
+async def test_registration_ask_is_refused_beside_a_credential_id_or_on_a_secret_turn(
+ monkeypatch: pytest.MonkeyPatch, conflict: str
+) -> None:
+ ctx = _tool_ctx(monkeypatch)
+ ctx.client_supports_credential_generation = True
+ extra: dict[str, Any] = {}
+ if conflict == "credential_id":
+ extra["credential_id"] = "cred_1"
+ else:
+ ctx.request_policy = _redacted_secret_policy("https://portal.example.com/login")
+
+ result = await _call_ask_tool(ctx, registration=_REGISTRATION, **extra)
+
+ assert result["ok"] is False
+ assert _sent_cards(ctx) == []
+
+
+def _readback_credential(credential_id: str, **update: Any) -> Credential:
+ return _make_credential(credential_id, "Portal test account").model_copy(
+ update={
+ "username": "[email protected]",
+ "created_by": "user-1",
+ "created_at": datetime.now(timezone.utc) + timedelta(seconds=5),
+ **update,
+ }
+ )
+
+
[email protected]("failure", ["rejected", "slow_prepare", "vault_error", "timeout"])
+async def test_registration_generate_failure_with_no_readback_match_reopens_the_card_once(
+ monkeypatch: pytest.MonkeyPatch, failure: str
+) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ _seed_registration_pause(cache)
+ late = _readback_credential("cred_late")
+ stale = _readback_credential("cred_stale", created_at=datetime.now(timezone.utc) - timedelta(hours=1))
+ other_user = _readback_credential("cred_other", created_by="user-2")
+ database = _registration_database(monkeypatch, late, readback=[other_user, stale])
+ vault = _RegistrationVault(
+ late,
+ reject=failure == "rejected",
+ fail=failure == "vault_error",
+ delay=0.2 if failure == "timeout" else 0,
+ prepare_delay=5 if failure == "slow_prepare" else 0,
+ )
+ vault.install(monkeypatch)
+ monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 1)
+ monkeypatch.setattr(copilot_routes, "_CLAIM_FINISH_MARGIN_SECONDS", 0.5)
+ monkeypatch.setattr(copilot_routes, "_READBACK_RESERVE_SECONDS", 0.5)
+
+ with capture_logs() as logs:
+ started = time.monotonic()
+ result = await _generate()
+ assert time.monotonic() - started < 2
+ expected = "rejected" if failure in ("rejected", "slow_prepare") else "unknown"
+ assert result.result == expected
+ assert result.credential_id is None
+ if expected == "rejected":
+ database.credentials.get_credentials.assert_not_awaited()
+ registration = _registration_record(cache)
+ assert registration is not None
+ assert (registration.attempted, registration.outcome) == (True, expected)
+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
+ database.workflow_params.record_pending_copilot_turn_credential_resume.assert_not_awaited()
+ with pytest.raises(HTTPException) as again:
+ await _generate()
+ assert again.value.status_code == status.HTTP_409_CONFLICT
+ assert len(vault.passwords) == (0 if expected == "rejected" else 1)
+ await asyncio.sleep(0.3)
+ assert vault.stored == (1 if failure == "timeout" else 0)
+ assert vault.hooks_run == (["cred_late"] if failure == "timeout" else [])
+ late_creates = [log["credential_id"] for log in logs if log["event"] == "copilot_credential_generation_late_create"]
+ assert late_creates == (["cred_late"] if failure == "timeout" else [])
+ for password in vault.passwords:
+ assert password not in json.dumps(logs, default=str)
+ assert password not in result.model_dump_json()
+ assert password not in str(cache.store)
+
+
[email protected](("create_seconds", "expected_id"), [(0.2, "cred_new"), (1.5, None)])
+async def test_registration_slow_create_connects_only_when_it_lands_before_the_wait_cutoff(
+ monkeypatch: pytest.MonkeyPatch, create_seconds: float, expected_id: str | None
+) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 2)
+ monkeypatch.setattr(copilot_routes, "_CLAIM_FINISH_MARGIN_SECONDS", 0.5)
+ monkeypatch.setattr(copilot_routes, "_READBACK_RESERVE_SECONDS", 0.5)
+ _seed_registration_pause(cache)
+ credential = _make_credential("cred_new", "Portal test account")
+ database = _registration_database(monkeypatch, credential)
+ vault = _RegistrationVault(credential, delay=create_seconds)
+ vault.install(monkeypatch)
+
+ result = await _generate()
+ with pytest.raises(HTTPException) as again:
+ await _generate()
+
+ assert again.value.status_code == status.HTTP_409_CONFLICT
+ assert len(vault.passwords) == 1
+ if expected_id is not None:
+ assert (result.result, result.credential_id) == ("connected", expected_id)
+ database.credentials.get_credentials.assert_not_awaited()
+ else:
+ assert (result.result, result.credential_id) == ("unknown", None)
+ database.credentials.get_credentials.assert_awaited_once()
+ registration = _registration_record(cache)
+ assert registration is not None and registration.outcome == "unknown"
+ await asyncio.sleep(create_seconds)
+ assert vault.stored == 1
+
+
[email protected](("vault_status", "expected"), [(422, "rejected"), (500, "unknown")])
+async def test_registration_vault_refusal_is_rejected_only_for_a_4xx(
+ monkeypatch: pytest.MonkeyPatch, vault_status: int, expected: str
+) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ _seed_registration_pause(cache)
+ credential = _readback_credential("cred_unrelated", created_by="user-2")
+ database = _registration_database(monkeypatch, credential, readback=[credential])
+ vault = _RegistrationVault(credential, vault_status=vault_status)
+ vault.install(monkeypatch)
+
+ with capture_logs() as logs:
+ result = await _generate()
+ with pytest.raises(HTTPException) as again:
+ await _generate()
+
+ assert (result.result, result.credential_id) == (expected, None)
+ assert again.value.status_code == status.HTTP_409_CONFLICT
+ assert len(vault.passwords) == 1
+ if expected == "rejected":
+ database.credentials.get_credentials.assert_not_awaited()
+ else:
+ database.credentials.get_credentials.assert_awaited()
+ registration = _registration_record(cache)
+ assert registration is not None and registration.outcome == expected
+ password = vault.passwords[0]
+ assert password not in json.dumps(logs, default=str)
+ assert password not in result.model_dump_json()
+ assert password not in str(cache.store)
+
+
[email protected]("readback", ["one", "two", "full_page", "other_password", "slow_read"])
+async def test_registration_unknown_save_connects_only_a_single_readback_match(
+ monkeypatch: pytest.MonkeyPatch, readback: str
+) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 1)
+ monkeypatch.setattr(copilot_routes, "_CLAIM_FINISH_MARGIN_SECONDS", 0.5)
+ _seed_registration_pause(cache)
+ match = _readback_credential("cred_match")
+ rows = {
+ "two": [match, _readback_credential("cred_twin")],
+ "full_page": [match, *(_readback_credential(f"cred_{i}", username=f"u{i}@example.com") for i in range(99))],
+ }.get(readback, [match])
+ database = _registration_database(monkeypatch, match, readback=rows)
+ vault = _RegistrationVault(
+ match,
+ fail=True,
+ stored_password="Another-create's-password-1" if readback == "other_password" else None,
+ read_delay=30 if readback == "slow_read" else 0,
+ )
+ vault.install(monkeypatch)
+
+ started = time.monotonic()
+ result = await _generate()
+
+ assert time.monotonic() - started < 5
+
+ assert len(vault.passwords) == 1
+ with pytest.raises(HTTPException) as again:
+ await _generate()
+ assert again.value.status_code == status.HTTP_409_CONFLICT
+ assert len(vault.passwords) == 1
+ if readback == "one":
+ assert (result.result, result.credential_id) == ("connected", "cred_match")
+ response = json.loads(cache.store[credential_response_cache_key("org-1", "chat-1", "turn-1")])
+ assert (response["action"], response["credential_id"]) == ("connected", "cred_match")
+ database.workflow_params.record_pending_copilot_turn_credential_resume.assert_awaited()
+ else:
+ assert (result.result, result.credential_id) == ("unknown", None)
+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
+ registration = _registration_record(cache)
+ assert registration is not None and registration.outcome == "unknown"
+
+
+ ("case", "expected_status"),
+ [
+ ("bad_token", status.HTTP_403_FORBIDDEN),
+ ("expired", status.HTTP_410_GONE),
+ ("consumed", status.HTTP_409_CONFLICT),
+ ("other_org", status.HTTP_404_NOT_FOUND),
+ ("not_a_registration_card", status.HTTP_409_CONFLICT),
+ ],
+)
+async def test_registration_generate_refuses_before_any_vault_call(
+ monkeypatch: pytest.MonkeyPatch, case: str, expected_status: int
+) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ _seed_registration_pause(
+ cache,
+ expires_at=datetime.now(timezone.utc) - timedelta(seconds=1) if case == "expired" else None,
+ registration=case != "not_a_registration_card",
+ )
+ if case == "consumed":
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-1",
+ resume_token="tok-1",
+ action="skip",
+ credential_id=None,
+ )
+ _registration_database(monkeypatch, _make_credential())
+ vault = _RegistrationVault(_make_credential())
+ vault.install(monkeypatch)
+
+ with pytest.raises(HTTPException) as refused:
+ await _generate(
+ resume_token="tok-wrong" if case == "bad_token" else "tok-1",
+ organization_id="org-2" if case == "other_org" else "org-1",
+ )
+
+ assert refused.value.status_code == expected_status
+ assert (vault.prepared, vault.passwords) == (0, [])
+
+
+async def test_registration_created_after_the_pause_ended_is_reported_but_not_connected(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ _seed_registration_pause(cache)
+ created = _make_credential("cred_late", "Portal test account")
+ _registration_database(monkeypatch, created)
+ vault = _RegistrationVault(created)
+ vault.install(monkeypatch)
+ store = vault.store
+
+ async def store_while_the_waiter_gives_up(
+ organization_id: str,
+ data: CreateCredentialRequest,
+ created_by: str | None,
+ service: str,
+ tasks: BackgroundTasks,
+ ) -> Credential:
+ cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")] = _encode_active_pause(
+ "tok-1", datetime.now(timezone.utc), consumed=True
+ )
+ return await store(organization_id, data, created_by, service, tasks)
+
+ monkeypatch.setattr(copilot_routes, "store_new_credential", store_while_the_waiter_gives_up)
+
+ result = await _generate()
+
+ assert result.result == "created_not_connected"
+ assert result.credential_id == "cred_late"
+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") not in cache.store
+
+
+async def test_registration_created_but_unrecorded_on_the_pause_is_reported_with_its_id(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ _seed_registration_pause(cache)
+ created = _make_credential("cred_saved", "Portal test account")
+ _registration_database(monkeypatch, created)
+ vault = _RegistrationVault(created)
+ vault.install(monkeypatch)
+ response_key = credential_response_cache_key("org-1", "chat-1", "turn-1")
+ cache_set = cache.set
+
+ async def set_failing_on_the_response(key: str, value: Any, ex: Any = None) -> None:
+ if key == response_key:
+ raise ConnectionError("cache unavailable")
+ await cache_set(key, value, ex)
+
+ monkeypatch.setattr(cache, "set", set_failing_on_the_response)
+
+ result = await _generate()
+
+ assert (result.result, result.credential_id) == ("created_not_connected", "cred_saved")
+ assert len(vault.passwords) == 1
+
+
[email protected]("failure", ["rejected", "vault_error"])
+async def test_registration_card_reopened_after_its_deadline_passed_mid_save_is_still_answerable(
+ monkeypatch: pytest.MonkeyPatch, failure: str
+) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ _seed_registration_pause(cache, expires_at=datetime.now(timezone.utc) + timedelta(milliseconds=50))
+ _registration_database(monkeypatch, _make_credential())
+ vault = _RegistrationVault(
+ _make_credential(), reject=failure == "rejected", fail=failure == "vault_error", delay=0.1
+ )
+ vault.install(monkeypatch)
+
+ result = await _generate()
+
+ record = _decode_active_pause(cache.store[credential_pause_active_key("org-1", "chat-1", "turn-1")])
+ assert record is not None and record.status == "pending"
+ assert result.expires_at == record.expires_at
+ assert record.expires_at > datetime.now(timezone.utc) + timedelta(
+ seconds=settings.WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS - 5
+ )
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-1",
+ resume_token="tok-1",
+ action="skip",
+ credential_id=None,
+ )
+ assert credential_response_cache_key("org-1", "chat-1", "turn-1") in cache.store
+
+
+async def test_registration_ask_with_a_charset_outside_the_enum_returns_an_error_the_model_sees(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _tool_ctx(monkeypatch)
+ ctx.client_supports_credential_generation = True
+
+ result = await _call_ask_tool(ctx, registration={**_REGISTRATION, "charset": "letters_and_spaces"})
+
+ assert result["ok"] is False
+ assert "registration.charset" in result["error"]
+ assert _sent_cards(ctx) == []
+ assert ctx.credential_pause_used is False
+
+
[email protected](params=[True, False], ids=["json", "console"])
+def rendered_logs(monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest) -> Iterator[io.StringIO]:
+ monkeypatch.setattr(settings, "JSON_LOGGING", request.param)
+ root = logging.getLogger()
+ saved_config = structlog.get_config()
+ saved_handlers, saved_level = root.handlers[:], root.level
+ stream = io.StringIO()
+ try:
+ setup_logger()
+ handler = root.handlers[0]
+ assert isinstance(handler, logging.StreamHandler)
+ handler.setStream(stream)
+ yield stream
+ finally:
+ structlog.configure(**saved_config)
+ root.handlers[:] = saved_handlers
+ root.setLevel(saved_level)
+
+
+async def test_registration_password_quoted_by_a_vault_error_is_scrubbed_from_rendered_logs_and_tracebacks(
+ monkeypatch: pytest.MonkeyPatch, rendered_logs: io.StringIO
+) -> None:
+ cache = _FakeCache()
+ monkeypatch.setattr(app._inst, "CACHE", cache, raising=False)
+ _seed_registration_pause(cache)
+ match = _readback_credential("cred_match")
+ _registration_database(monkeypatch, match, readback=[match])
+ vault = _RegistrationVault(match, fail=True, read_fails=True)
+ vault.install(monkeypatch)
+
+ result = await _generate()
+
+ [password] = vault.passwords
+ rendered = rendered_logs.getvalue()
+ assert result.result == "unknown"
+ assert "vault create failed" in rendered
+ assert "copilot_credential_generation_readback_failed" in rendered
+ assert "in an unexpected shape" in rendered
+ assert "Traceback" in rendered
+ assert password not in rendered
+
+
[email protected]("answer", ["skip", "timeout"])
+async def test_registration_card_reopened_after_an_unconfirmed_save_reports_it_when_skipped_or_unanswered(
+ monkeypatch: pytest.MonkeyPatch, answer: str
+) -> None:
+ cache = _FakeCache()
+ ctx = _tool_ctx(monkeypatch, cache)
+ ctx.client_supports_credential_generation = True
+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=1)
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS", 1)
+ _registration_database(monkeypatch, _make_credential())
+ vault = _RegistrationVault(_make_credential(), fail=True)
+ vault.install(monkeypatch)
+
+ async def generate_then_answer(
+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
+ ) -> bool:
+ if frame.type != WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
+ return True
+ assert (await _generate(resume_token=frame.resume_token)).result == "unknown"
+ if answer == "skip":
+ await resolve_credential_pause(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-1",
+ resume_token=frame.resume_token,
+ action="skip",
+ credential_id=None,
+ )
+ return True
+
+ ctx.stream.send.side_effect = generate_then_answer
+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)
+
+ assert result["status"] == ("skipped" if answer == "skip" else "unanswered")
+ assert result["registration_outcome"] == "unknown"
+ assert "next" not in result
+ assert len(vault.passwords) == 1
+ assert vault.passwords[0] not in json.dumps(result)
+
+
+async def test_registration_card_that_ran_out_mid_create_reports_unknown(monkeypatch: pytest.MonkeyPatch) -> None:
+ cache = _FakeCache()
+ ctx = _tool_ctx(monkeypatch, cache)
+ ctx.client_supports_credential_generation = True
+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=1)
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_CREDENTIAL_PAUSE_TIMEOUT_SECONDS", 1)
+ monkeypatch.setattr(credential_pause_module, "MANUAL_SIGN_IN_CLAIM_SECONDS", 1)
+
+ async def claim_and_abandon(
+ frame: WorkflowCopilotCredentialRequiredUpdate | WorkflowCopilotCredentialPauseResolvedUpdate,
+ ) -> bool:
+ if frame.type == WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED:
+ await claim_credential_generation(
+ cache,
+ organization_id="org-1",
+ workflow_copilot_chat_id="chat-1",
+ turn_id="turn-1",
+ resume_token=frame.resume_token,
+ )
+ return True
+
+ ctx.stream.send.side_effect = claim_and_abandon
+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)
+
+ assert result["status"] == "unanswered"
+ assert result["registration_outcome"] == "unknown"
+
+
+async def test_registration_outcome_from_an_earlier_ask_is_not_reported_on_the_next(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ctx = _tool_ctx(monkeypatch, _FakeCache())
+ ctx.client_supports_credential_generation = True
+ ctx.credential_registration_outcome = "rejected"
+ ctx.client_supports_credential_pause_recovery = False
+ ctx.stream.is_disconnected.return_value = True
+
+ result = await _call_ask_tool(ctx, registration=_REGISTRATION)
+
+ assert result["status"] == "unanswered"
+ assert "registration_outcome" not in result
diff --git a/tests/unit/test_copilot_credential_pause_reload.py b/tests/unit/test_copilot_credential_pause_reload.py
index ac616f3b0..b2e428790 100644
--- a/tests/unit/test_copilot_credential_pause_reload.py
+++ b/tests/unit/test_copilot_credential_pause_reload.py
@@ -12,10 +12,19 @@ from skyvern.forge.sdk.copilot import credential_pause as pauses
from skyvern.forge.sdk.copilot.config import CopilotConfig
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, _ground_user_provided_sites
from skyvern.forge.sdk.copilot.tools.credential_fill import _request_credential
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.routes import workflow_copilot as routes
from skyvern.forge.sdk.schemas.browser_profiles import BrowserProfile
+from skyvern.forge.sdk.schemas.workflow_copilot import CredentialRegistration
from tests.unit.test_copilot_ask_user import setup_question_chat
-from tests.unit.test_copilot_credential_pause import _FakeCache, _make_credential, _make_stream
+from tests.unit.test_copilot_credential_pause import (
+ _REGISTRATION,
+ _FakeCache,
+ _make_credential,
+ _make_stream,
+ _RegistrationVault,
+)
@pytest.mark.asyncio
@@ -406,3 +415,79 @@ async def test_a_done_whose_pause_ended_while_saving_discards_the_profile(sqlite
)
response_key = pauses.credential_response_cache_key("org", card.workflow_copilot_chat_id, card.turn_id)
assert response_key not in cache.store
+
+
+async def test_registration_card_survives_a_reload_with_its_one_generate_attempt_recorded(sqlite_engine, monkeypatch):
+ _, client, ctx, _ = await setup_question_chat(sqlite_engine, monkeypatch)
+ monkeypatch.setattr(app, "CACHE", _FakeCache())
+ monkeypatch.setattr(pauses, "CREDENTIAL_RESPONSE_POLL_SECONDS", 0.01)
+ monkeypatch.setattr(routes, "RECONCILE_ABANDON_AFTER_SECONDS", -1)
+ monkeypatch.setattr(app.DATABASE, "credentials", SimpleNamespace(get_credentials=AsyncMock(return_value=[])))
+ emitted = _sign_in_card_context(ctx, recovery=True)
+ ctx.client_supports_credential_generation = True
+ recovery_token = "a" * 64
+ vault = _RegistrationVault(_make_credential(), fail=True)
+ vault.install(monkeypatch)
+ recovered_while_saving: list[list] = []
+ fail_store = vault.store
+
+ async def store_and_reload(*args):
+ recovered_while_saving.append(
+ await pauses.pending_credential_requests("org", ctx.workflow_copilot_chat_id, ["turn"], recovery_token)
+ )
+ return await fail_store(*args)
+
+ monkeypatch.setattr(routes, "store_new_credential", store_and_reload)
+ api = client._transport.app
+ api.dependency_overrides[routes.org_auth_service.get_current_org_for_credential_routes] = api.dependency_overrides[
+ routes.org_auth_service.get_current_org
+ ]
+ api.dependency_overrides[routes.org_auth_service.get_current_user_id_or_none] = lambda: "user-1"
+ api.add_api_route("/credential-generate", routes.workflow_copilot_credential_generate, methods=["POST"])
+
+ async def history() -> list[dict]:
+ response = await client.get(
+ "/history",
+ params={"workflow_copilot_chat_id": ctx.workflow_copilot_chat_id},
+ headers={"X-Copilot-Credential-Recovery-Token": recovery_token},
+ )
+ return response.json()["pending_credential_requests"]
+
+ async with client:
+ invocation = asyncio.create_task(
+ _request_credential(
+ "https://portal.example.com/signup",
+ "Create the test account",
+ ctx,
+ registration=CredentialRegistration(**_REGISTRATION),
+ )
+ )
+ try:
+ await asyncio.wait_for(emitted.wait(), 1)
+ [raised] = await history()
+ assert raised["reason"] == "credential_registration"
+ assert raised["registration"]["credential_name"] == "Portal test account"
+ assert raised["registration"]["attempted"] is False
+ body = {
+ "workflow_copilot_chat_id": raised["workflow_copilot_chat_id"],
+ "turn_id": raised["turn_id"],
+ "resume_token": raised["resume_token"],
+ }
+
+ with skyvern_context.scoped(SkyvernContext()):
+ generated = await client.post("/credential-generate", json=body)
+
+ assert generated.json()["result"] == "unknown"
+ [[saving]] = recovered_while_saving
+ assert saving.registration is not None and saving.registration.attempted is True
+ [reopened] = await history()
+ assert reopened["registration"]["attempted"] is True
+ assert reopened["registration"]["outcome"] == "unknown"
+ with skyvern_context.scoped(SkyvernContext()):
+ assert (await client.post("/credential-generate", json=body)).status_code == 409
+ assert len(vault.passwords) == 1
+ assert not invocation.done()
+ finally:
+ invocation.cancel()
+ await asyncio.gather(invocation, return_exceptions=True)
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index b608f04c0..797f6565b 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -343,7 +343,7 @@ def test_non_paused_watchdog_exit_still_classifies_as_a_failed_run() -> None:
source_tool="run_blocks_and_collect_debug",
result={
"ok": False,
- "error": "The run exceeded the 600s absolute ceiling while still showing progress.",
+ "error": "The run did not reach a terminal status within the 600s absolute ceiling.",
"data": {
"workflow_run_id": "wr_ceiling",
"overall_status": "running",
@@ -3474,6 +3474,9 @@ async def test_completed_missing_output_complete_fact_packet_reaches_ordinary_re
modified_at=now,
)
run_workflow = SimpleNamespace(
+ workflow_id="wf_run_snapshot",
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ version=1,
organization_id=ctx.organization_id,
created_by=None,
modified_at=datetime(2026, 4, 21, 12, 0),
@@ -3484,12 +3487,14 @@ async def test_completed_missing_output_complete_fact_packet_reaches_ordinary_re
)
run = SimpleNamespace(
workflow_run_id="wr_completed_missing_output",
+ workflow_id="wf_run_snapshot",
workflow_permanent_id=ctx.workflow_permanent_id,
browser_session_id="pbs_completed_missing_output",
status="completed",
failure_reason=None,
created_at=datetime(2026, 4, 21, 12, 0),
trigger_type=None,
+ workflow_definition_sha256=None,
)
block = WorkflowRunBlock(
workflow_run_block_id="wrb_completed_missing_output",
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 15fccdd47..b961281df 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -599,7 +599,7 @@ def test_record_run_blocks_result_resets_stale_verified_terminal_proposal_latch(
def test_record_run_blocks_result_keeps_failure_when_watchdog_cancel_without_timeout() -> None:
- """Stagnation/ceiling cancels mid-session must still set last_test_ok=False
+ """Ceiling cancels mid-session must still set last_test_ok=False
so the failed-test nudge can fire — only a coincident total timeout softens
to ``None`` for the unvalidated WIP rescue path."""
ctx = _fresh_ctx_for_record()
diff --git a/tests/unit/test_copilot_fill_credential_field.py b/tests/unit/test_copilot_fill_credential_field.py
index 05fad3c68..4ff79f10a 100644
--- a/tests/unit/test_copilot_fill_credential_field.py
+++ b/tests/unit/test_copilot_fill_credential_field.py
@@ -1193,21 +1193,26 @@ class TestCredentialFillInCallSubmit:
assert "matches 2 controls" in result["data"]["submit_skipped"]
@pytest.mark.asyncio
- async def test_a_failed_submit_click_never_costs_the_fill(self, monkeypatch: pytest.MonkeyPatch) -> None:
- page = _FakePage(click_error=RuntimeError("no element matched #verifyButton for 123456"))
- _wire_impl(monkeypatch, page, secret_value="123456")
+ @pytest.mark.parametrize(("field", "secret"), [("totp", "123456"), ("password", "Generated-Pa55word!xyz")])
+ async def test_a_failed_submit_click_never_costs_the_fill(
+ self, monkeypatch: pytest.MonkeyPatch, field: str, secret: str
+ ) -> None:
+ page = _FakePage(click_error=RuntimeError(f"no element matched #verifyButton for {secret}"))
+ _wire_impl(monkeypatch, page, secret_value=secret)
ctx = _ctx()
- result = await tools_module._fill_credential_field_impl(ctx, "#totpCode", "cred_123", "totp", "#verifyButton")
+ result = await tools_module._fill_credential_field_impl(ctx, f"#{field}", "cred_123", field, "#verifyButton")
assert result["ok"] is True
- assert result["data"]["typed_length"] == 6
- assert "123456" not in json.dumps(result)
+ assert result["data"]["typed_length"] == len(secret)
+ assert secret not in json.dumps(result)
assert "[REDACTED_SECRET]" in result["data"]["submit_error"]
# The click raised, so whether it reached the page is unknown. Saying only "not submitted"
# would read as safe to retry, and a retry spends a second code.
assert result["data"]["submitted"] is False
assert result["data"]["submit_uncertain"] is True
+ assert "may already have been submitted" in result["data"]["submit_error"]
+ assert page.click_calls == [("#verifyButton",)]
assert [entry["tool_name"] for entry in ctx.scout_trajectory] == ["fill_credential_field"]
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.py
index c281cbfff..741be3b10 100644
--- a/tests/unit/test_copilot_frontier.py
+++ b/tests/unit/test_copilot_frontier.py
@@ -160,6 +160,12 @@ class _FakeWorkflow:
return copy.deepcopy(self) if deep else _FakeWorkflow(self.workflow_definition)
+async def _restore_the_offered_proposal(ctx: CopilotContext, **_: object) -> None:
+ # The route only admits the Test action over a pending proposal, which restore stages.
+ ctx.staged_workflow = _FakeWorkflow(_wf_def())
+ ctx.staged_workflow_yaml = ctx.workflow_yaml
+
+
class _FakeStream:
async def is_disconnected(self) -> bool:
return False
@@ -4523,6 +4529,7 @@ async def test_noncompleted_test_result_handoff_survives_provider_input_merge_an
else AsyncMock(return_value=run_result)
)
monkeypatch.setattr(agent_module, "run_workflow_end_to_end", run)
+ monkeypatch.setattr(agent_module, "restore_pending_workflow_proposal", _restore_the_offered_proposal)
monkeypatch.setattr(agent_module, "_resolve_live_browser_session_id", AsyncMock(return_value=None))
monkeypatch.setattr("agents.mcp.MCPServerManager", FakeMCPServerManager)
monkeypatch.setattr(
@@ -4827,6 +4834,7 @@ async def test_completed_test_result_handoff_uses_ordinary_acting_agent_surface(
return result
monkeypatch.setattr(agent_module, "run_workflow_end_to_end", fake_run)
+ monkeypatch.setattr(agent_module, "restore_pending_workflow_proposal", _restore_the_offered_proposal)
monkeypatch.setattr(agent_module, "_resolve_live_browser_session_id", AsyncMock(return_value=None))
monkeypatch.setattr("agents.mcp.MCPServerManager", FakeMCPServerManager)
monkeypatch.setattr(
diff --git a/tests/unit/test_copilot_locator_inspection.py b/tests/unit/test_copilot_locator_inspection.py
index eae56846e..3ccbec17f 100644
--- a/tests/unit/test_copilot_locator_inspection.py
+++ b/tests/unit/test_copilot_locator_inspection.py
@@ -43,6 +43,8 @@ from skyvern.forge.sdk.copilot.tools.locator_inspection import (
)
from skyvern.forge.sdk.copilot.tools.run_execution import build_test_evidence_packet
from skyvern.forge.sdk.schemas.workflow_runs import WorkflowRunBlock
+from skyvern.forge.sdk.workflow.models.block import CodeBlock
+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
from skyvern.schemas.workflows import BlockType
from tests.unit.copilot_test_helpers import (
SENSITIVE_DISCLOSURE_WITHHOLDING_ARMS,
@@ -905,6 +907,7 @@ async def test_prior_run_result_uses_its_exact_failed_row_and_workflow_for_typed
failure_reason="click failed",
created_at=datetime(2026, 4, 21, 12, 0),
trigger_type=None,
+ workflow_definition_sha256=None,
)
now = datetime.now(UTC)
block = WorkflowRunBlock(
@@ -921,15 +924,24 @@ async def test_prior_run_result_uses_its_exact_failed_row_and_workflow_for_typed
modified_at=now,
)
workflow = SimpleNamespace(
+ workflow_id="w_prior",
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ version=1,
created_by=None,
modified_at=datetime(2026, 4, 21, 12, 0),
workflow_definition=SimpleNamespace(
blocks=[
- {
- "label": "click_submit",
- "block_type": "code",
- "code": 'await page.locator("#submit").click()',
- }
+ CodeBlock(
+ label="click_submit",
+ code='await page.locator("#submit").click()',
+ output_parameter=OutputParameter(
+ output_parameter_id="op_click_submit",
+ key="click_submit_output",
+ workflow_id="w_prior",
+ created_at=now,
+ modified_at=now,
+ ),
+ )
]
),
)
diff --git a/tests/unit/test_copilot_narration.py b/tests/unit/test_copilot_narration.py
index f3a3ad3c9..20e5a74e6 100644
--- a/tests/unit/test_copilot_narration.py
+++ b/tests/unit/test_copilot_narration.py
@@ -94,13 +94,12 @@ async def test_actual_run_polling_bridge_emits_progress_while_action_is_pending(
nonlocal reads
reads += 1
if reads == 1:
- return SimpleNamespace(status="running", modified_at=stamp), stamp, stamp
+ return SimpleNamespace(status="running", modified_at=stamp), stamp
if reads == 2:
- return SimpleNamespace(status="running", modified_at=stamp), stamp, stamp + timedelta(seconds=1)
+ return SimpleNamespace(status="running", modified_at=stamp), stamp + timedelta(seconds=1)
await release.wait()
return (
SimpleNamespace(status="completed", modified_at=stamp, failure_reason=None),
- stamp,
stamp + timedelta(seconds=2),
)
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index a1be8dc0b..0e7b0b28f 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -360,7 +360,7 @@ def test_watchdog_model_facing_text_is_allowed_as_a_run_result_reply() -> None:
request_policy=None,
response_type="REPLY",
user_response=asyncio.run(
- _watchdog_error_message("stagnation", SimpleNamespace(), "wr_123", None, 300, dispatch_to_worker=True)
+ _watchdog_error_message("ceiling", SimpleNamespace(), "wr_123", None, 300, dispatch_to_worker=True)
),
output_kind=CopilotOutputKind.WORKFLOW_RUN_RESULT,
)
diff --git a/tests/unit/test_copilot_output_utils.py b/tests/unit/test_copilot_output_utils.py
index 6f995faf7..b5ac583dd 100644
--- a/tests/unit/test_copilot_output_utils.py
+++ b/tests/unit/test_copilot_output_utils.py
@@ -1411,25 +1411,25 @@ class TestFormatToolResultForUser:
result = {
"ok": False,
"error": (
- "The run has not made progress. Run ID: wr_stalled. Outcome is uncertain. "
- "Do NOT re-invoke block-running tools without first calling get_run_results."
+ "The run did not reach a terminal status within the 1190s absolute ceiling. Run ID: wr_stalled. "
+ "Outcome is uncertain. Do NOT re-invoke block-running tools without first calling get_run_results."
),
"data": {
"failure_reason": (
- "The run stopped after no observable progress for 120s. Run ID: wr_stalled. Outcome is uncertain."
+ "The run did not finish within the 1190s absolute ceiling. Run ID: wr_stalled. Outcome is uncertain."
),
"control_signal": {
- "kind": "watchdog_stagnation",
- "user_facing_summary": "The run stopped after no observable progress for 120s.",
+ "kind": "watchdog_ceiling",
+ "user_facing_summary": "The run did not finish within the 1190s absolute ceiling.",
},
- "user_facing_summary": "The run stopped after no observable progress for 120s.",
+ "user_facing_summary": "The run did not finish within the 1190s absolute ceiling.",
},
}
summary = self._format("run_blocks_and_collect_debug", result)
detail = summarize_tool_result_detail(result, tool_name="run_blocks_and_collect_debug")
- assert summary == "The run stopped after no observable progress for 120s."
+ assert summary == "The run did not finish within the 1190s absolute ceiling."
assert detail == summary
assert "wr_stalled" not in summary
assert "get_run_results" not in detail
diff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.py
index e534dc869..8a93c061a 100644
--- a/tests/unit/test_copilot_per_tool_budget.py
+++ b/tests/unit/test_copilot_per_tool_budget.py
@@ -208,11 +208,11 @@ def test_watchdog_cancel_with_stale_challenge_markup_is_not_promoted() -> None:
}
result = {
"ok": False,
- "error": "Run canceled after 90s of stagnation while a Cloudflare interstitial was displayed.",
+ "error": "Run canceled at the safety ceiling while a Cloudflare interstitial was displayed.",
"data": {
"workflow_run_id": "wr_cancelled",
"overall_status": "canceled",
- "failure_reason": "Run canceled after stagnation.",
+ "failure_reason": "Run canceled at the safety ceiling.",
"blocks": [],
},
}
diff --git a/tests/unit/test_copilot_positive_evidence_carriers.py b/tests/unit/test_copilot_positive_evidence_carriers.py
index 43260a32c..511600020 100644
--- a/tests/unit/test_copilot_positive_evidence_carriers.py
+++ b/tests/unit/test_copilot_positive_evidence_carriers.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import json
import os
from collections.abc import Sequence
from dataclasses import FrozenInstanceError
@@ -12,6 +13,7 @@ from unittest.mock import AsyncMock
import pytest
from skyvern.forge.sdk.artifact.models import ArtifactType
+from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.build_test_outcome import ChallengeEffects, challenge_notices
from skyvern.forge.sdk.copilot.challenge_evidence import stamp_challenge_frame_fact
from skyvern.forge.sdk.copilot.completion_output_grounding import (
@@ -51,6 +53,9 @@ from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import _register_code_block_secret
from tests.unit.copilot_test_helpers import (
DISPATCHED_NAV_ONLY_HTML,
+ harness_run,
+ historical_code_version,
+ install_historical_run,
make_completion_criterion,
make_copilot_ctx,
)
@@ -1068,8 +1073,12 @@ async def test_origin_registry_recovers_when_a_later_terminal_read_finds_the_han
browser_session_id=None,
created_at=datetime(2026, 4, 21, 12, 0),
trigger_type=None,
+ workflow_definition_sha256=None,
)
workflow = SimpleNamespace(
+ workflow_id="wf_origin",
+ workflow_permanent_id=ctx.workflow_permanent_id,
+ version=1,
created_by=None,
modified_at=datetime(2026, 4, 21, 12, 0),
workflow_definition=SimpleNamespace(parameters=[], blocks=[]),
@@ -2174,3 +2183,132 @@ def test_runtime_output_scalar_still_credits_normally() -> None:
# untouched, so the reject does not swallow the legitimate path.
verdict = grade_requested_output_criteria(_star_ctx(), [_exact_value_criterion("22600")], _star_snapshot())[0]
assert verdict.state == "satisfied"
+
+
+_EXECUTED_SECRET = "Rg7vQ2mXpL9sKd4TzW8nYb3HcJ5fUa6E"
+
+
+async def _run_results_page(ctx: CopilotContext, **arguments: object) -> str:
+ return await tools_module.get_run_results_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="get_run_results"), json.dumps(arguments)
+ )
+
+
+async def test_a_registered_secret_in_executed_source_or_defaults_reaches_no_page_or_packet(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Dense enough that the receipt's cut lands inside an occurrence, as it would if scrubbing ran after cutting.
+ secret_lines = [f'v{index} = "{_EXECUTED_SECRET}"' for index in range(400)]
+ code = "\n".join([f'token = "{_EXECUTED_SECRET}" + legacy_applicant_name', *secret_lines]) + "\n"
+ # The secret straddles the 120-character label cut.
+ label = "greet_" + "x" * 104 + _EXECUTED_SECRET
+ install_historical_run(
+ monkeypatch,
+ historical_code_version({label: code}, default_value=_EXECUTED_SECRET),
+ failed_label=label,
+ )
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *_args: None)
+ ctx = make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml="")
+ ctx.secret_scrub_values = [_EXECUTED_SECRET]
+
+ first_page = await _run_results_page(ctx, workflow_run_id="wr-1")
+ detail_page = await _run_results_page(ctx, workflow_run_id="wr-1", row_keys=[f"wrb_{label}"])
+ hydrated = await run_execution_module.hydrate_prior_run_packet(ctx, workflow_run_id="wr-1")
+
+ source = json.loads(first_page)["data"]["execution_source"]
+ assert source["disposition"] == "available"
+ assert source["failed_statement"].endswith("+ legacy_applicant_name")
+ assert source["failed_block_code_truncated"] is True
+ assert hydrated is not None and hydrated["run"]["execution_source"] == source
+ fragments = {_EXECUTED_SECRET[start : start + 8] for start in range(len(_EXECUTED_SECRET) - 7)}
+ for projection in (first_page, detail_page, json.dumps(hydrated)):
+ assert not [fragment for fragment in fragments if fragment in projection]
+
+
+async def test_a_scrubbed_multiline_secret_withholds_the_failed_statement_instead_of_naming_a_shifted_line(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ multiline_secret = "-----BEGIN KEY-----\nRg7vQ2mXpL9sKd4T\nzW8nYb3HcJ5fUa6E\n-----END KEY-----"
+ filler = "".join(f"unrelated_{index} = {index}\n" for index in range(5))
+ code = f'key = """{multiline_secret}"""\ngreeting = "Hello " + legacy_applicant_name\n{filler}'
+ install_historical_run(
+ monkeypatch,
+ historical_code_version({"greet_applicant": code}),
+ failed_label="greet_applicant",
+ failing_line=5,
+ )
+ ctx = make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml="")
+ ctx.secret_scrub_values = [multiline_secret]
+
+ result = await run_execution_module._get_run_results({"workflow_run_id": "wr-1"}, ctx)
+
+ source = result["data"]["execution_source"]
+ assert source["disposition"] == "available"
+ assert "failed_statement" not in source
+ assert "legacy_applicant_name" in source["failed_block_code"]
+ assert "Rg7vQ2mX" not in json.dumps(source)
+
+
+async def test_a_pathological_executed_version_keeps_identity_and_statement_within_the_receipt_budget(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ statement = "greeting = " + " + ".join(["legacy_applicant_name"] * 60)
+ code = statement + "\n" + "\n".join(f"step_{index} = {index}" for index in range(2_000)) + "\n"
+ install_historical_run(
+ monkeypatch,
+ historical_code_version(
+ {"greet_applicant": code}, parameter_keys=[f"{index:03d}_{'k' * 300}" for index in range(300)]
+ ),
+ failed_label="greet_applicant",
+ )
+
+ result = await run_execution_module._get_run_results(
+ {"workflow_run_id": "wr-1"}, make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml="")
+ )
+
+ source = result["data"]["execution_source"]
+ assert len(json.dumps(source)) <= 8_000
+ assert (source["workflow_run_id"], source["workflow_id"], source["workflow_version"]) == ("wr-1", "wf-1", 3)
+ assert source["failed_statement"] == statement[:600]
+ assert source["failed_statement_truncated"] is True
+ assert source["declared_parameter_keys_omitted"] > 0
+ assert all(len(key) <= 120 for key in source["declared_parameter_keys"])
+
+
+async def test_escaped_text_and_labels_sharing_a_cut_stay_within_the_receipt_and_are_counted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # JSON escapes each supplementary-plane character to 12 characters, so code-point caps alone overflow.
+ failed_label = "\U0001f600" * 200
+ shared_prefix = "x" * 120
+ statement = "greeting = " + "\U0001f600" * 700
+ install_historical_run(
+ monkeypatch,
+ historical_code_version({failed_label: statement + "\n" + "step = 1\n" * 2_000}),
+ failed_label=failed_label,
+ )
+
+ result = await run_execution_module._get_run_results(
+ {"workflow_run_id": "wr-1"}, make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml="")
+ )
+
+ source = result["data"]["execution_source"]
+ assert len(json.dumps(source)) <= 8_000
+ assert len(json.dumps(source["failed_block_label"])) - 2 <= 120
+ assert len(json.dumps(source["failed_statement"])) - 2 <= 600
+ assert source["failed_statement_truncated"] is True
+
+ colliding = run_execution_module._run_version_source(
+ make_copilot_ctx(workflow_permanent_id="wpid-1", workflow_yaml=""),
+ harness_run("wr-1"),
+ historical_code_version({shared_prefix + "_first": "pass\n", shared_prefix + "_second": "pass\n"}),
+ failed_block_label=None,
+ failing_line=None,
+ requested_block_labels=[shared_prefix + "_first", shared_prefix + "_second"],
+ )
+ assert list(colliding["block_code_sha256"]) == [shared_prefix]
+ assert colliding["code_block_labels_omitted"] == 1
diff --git a/tests/unit/test_copilot_watchdog.py b/tests/unit/test_copilot_watchdog.py
index 78c04605a..acc918e92 100644
--- a/tests/unit/test_copilot_watchdog.py
+++ b/tests/unit/test_copilot_watchdog.py
@@ -1,33 +1,10 @@
-"""Tests for the SKY-9163 progress-based watchdog inside
-``_run_blocks_and_collect_debug``.
-
-The full function does too much setup (prepare_workflow, execute_workflow,
-parameter-binding invariants) to unit-test end-to-end cheaply. Instead we
-target the isolated watchdog surface:
-
-- ``_progress_marker`` — marker stability and field-change sensitivity.
-- ``_read_progress_sources`` — correct delegation + graceful handling of
- DB failures.
-- ``_watchdog_error_message`` — the regression-guard strings (no
- "timed out", reconciliation-instruction, per-reason body).
-
-Those three are where the SKY-9163 correctness properties live:
-
-1. A stale marker must be exactly equal across two polls when nothing
- changed in the DB (otherwise the watchdog would false-reset on every
- poll, making stagnation detection impossible).
-2. Any change in ``run.status`` / ``run.modified_at`` / ``step_ts`` /
- ``block_ts`` must produce a new marker (otherwise the watchdog would
- false-trip on a progressing run).
-3. The error messages must not read as retry-invites — that was the
- original bug. "timed out" / "likely stuck repeating failing actions"
- are the exact phrases the LLM used to read as "try again".
-"""
+"""Tests for the run poll loop inside ``_run_blocks_and_collect_debug``: its exits, error strings and cancellation."""
from __future__ import annotations
import asyncio
import time
+from collections.abc import Callable
from datetime import UTC, datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Any
@@ -49,11 +26,8 @@ from skyvern.forge.sdk.copilot.repair_origin_run import (
)
from skyvern.forge.sdk.copilot.tools import (
RUN_BLOCKS_SAFETY_CEILING_SECONDS,
- RUN_BLOCKS_STAGNATION_WINDOW_SECONDS,
WatchdogExitReason,
- _any_quiet_block_requested,
_fallback_page_info,
- _progress_marker,
_read_progress_sources,
_run_blocks_and_collect_debug,
_watchdog_error_message,
@@ -73,9 +47,6 @@ from tests.unit.copilot_test_helpers import make_copilot_ctx, run_result_action_
def _fake_run(status: str = "running", modified_at: datetime | None = None) -> Any:
- """A bare-minimum stand-in for ``WorkflowRun`` — the marker only reads
- ``.status`` and ``.modified_at``.
- """
return SimpleNamespace(
status=status,
modified_at=modified_at or datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc),
@@ -84,62 +55,6 @@ def _fake_run(status: str = "running", modified_at: datetime | None = None) -> A
)
-# ---------------------------------------------------------------------------
-# _progress_marker: stability + per-field sensitivity.
-# ---------------------------------------------------------------------------
-
-
-def test_progress_marker_stable_for_identical_inputs() -> None:
- """If the DB reports identical values on two successive polls, the marker
- must compare equal. A marker that drifts on repeated reads would make the
- stagnation window unreachable."""
- run = _fake_run()
- step_ts = datetime(2026, 4, 21, 12, 0, 30, tzinfo=timezone.utc)
- block_ts = datetime(2026, 4, 21, 12, 0, 31, tzinfo=timezone.utc)
-
- m1 = _progress_marker(run, step_ts, block_ts)
- m2 = _progress_marker(run, step_ts, block_ts)
-
- assert m1 == m2
-
-
-def test_progress_marker_changes_on_run_status() -> None:
- run1 = _fake_run(status="running")
- run2 = _fake_run(status="queued")
- assert _progress_marker(run1, None, None) != _progress_marker(run2, None, None)
-
-
-def test_progress_marker_changes_on_run_modified_at() -> None:
- run1 = _fake_run(modified_at=datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc))
- run2 = _fake_run(modified_at=datetime(2026, 4, 21, 12, 0, 1, tzinfo=timezone.utc))
- assert _progress_marker(run1, None, None) != _progress_marker(run2, None, None)
-
-
-def test_progress_marker_changes_on_step_ts() -> None:
- run = _fake_run()
- t1 = datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc)
- t2 = datetime(2026, 4, 21, 12, 0, 5, tzinfo=timezone.utc)
- assert _progress_marker(run, t1, None) != _progress_marker(run, t2, None)
-
-
-def test_progress_marker_changes_on_block_ts() -> None:
- run = _fake_run()
- t1 = datetime(2026, 4, 21, 12, 0, 0, tzinfo=timezone.utc)
- t2 = datetime(2026, 4, 21, 12, 0, 5, tzinfo=timezone.utc)
- assert _progress_marker(run, None, t1) != _progress_marker(run, None, t2)
-
-
-def test_progress_marker_tolerates_none_run() -> None:
- """A transient DB read failure can return ``run=None``. The marker must
- still be hashable and comparable."""
- m_none = _progress_marker(None, None, None)
- assert m_none == (None, None, None, None)
-
- # Two consecutive failed reads produce equal markers → stagnation clock
- # keeps ticking (the right behavior when we can't confirm progress).
- assert _progress_marker(None, None, None) == _progress_marker(None, None, None)
-
-
# ---------------------------------------------------------------------------
# _read_progress_sources: delegation + graceful DB-failure handling.
# ---------------------------------------------------------------------------
@@ -149,25 +64,23 @@ class _FakeTasksRepo:
def __init__(
self,
*,
- step_ts: datetime | None = None,
block_ts: datetime | None = None,
raise_on_call: Exception | None = None,
) -> None:
- self.step_ts = step_ts
self.block_ts = block_ts
self.raise_on_call = raise_on_call
self.call_count = 0
- async def get_workflow_run_progress_timestamps(
+ async def get_workflow_run_block_progress_timestamp(
self,
*,
workflow_run_id: str,
organization_id: str | None = None,
- ) -> tuple[datetime | None, datetime | None]:
+ ) -> datetime | None:
self.call_count += 1
if self.raise_on_call is not None:
raise self.raise_on_call
- return self.step_ts, self.block_ts
+ return self.block_ts
class _FakeWorkflowRunsRepo:
@@ -203,19 +116,14 @@ async def test_read_progress_sources_returns_run_and_timestamps(
from skyvern.forge import app as forge_app
run = _fake_run()
- step_ts = datetime(2026, 4, 21, 12, 0, 10, tzinfo=timezone.utc)
block_ts = datetime(2026, 4, 21, 12, 0, 11, tzinfo=timezone.utc)
db = _FakeDatabase(
- tasks=_FakeTasksRepo(step_ts=step_ts, block_ts=block_ts),
+ tasks=_FakeTasksRepo(block_ts=block_ts),
workflow_runs=_FakeWorkflowRunsRepo(run=run),
)
monkeypatch.setattr(forge_app, "DATABASE", db)
- read_run, read_step_ts, read_block_ts = await _read_progress_sources(_FakeCtx(), "wr_1")
-
- assert read_run is run
- assert read_step_ts == step_ts
- assert read_block_ts == block_ts
+ assert await _read_progress_sources(_FakeCtx(), "wr_1") == (run, block_ts)
@pytest.mark.asyncio
@@ -227,25 +135,20 @@ async def test_read_progress_sources_swallows_workflow_run_errors(
from skyvern.forge import app as forge_app
db = _FakeDatabase(
- tasks=_FakeTasksRepo(step_ts=None, block_ts=None),
+ tasks=_FakeTasksRepo(block_ts=None),
workflow_runs=_FakeWorkflowRunsRepo(raise_on_call=RuntimeError("DB flake")),
)
monkeypatch.setattr(forge_app, "DATABASE", db)
- read_run, read_step_ts, read_block_ts = await _read_progress_sources(_FakeCtx(), "wr_1")
-
- assert read_run is None
- assert read_step_ts is None
- assert read_block_ts is None
+ assert await _read_progress_sources(_FakeCtx(), "wr_1") == (None, None)
@pytest.mark.asyncio
async def test_read_progress_sources_swallows_progress_timestamps_errors(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """A DB read failure on the aggregate timestamps must also not crash — the
- caller still gets the run (if readable) and ``None`` for the timestamps.
- """
+ """A DB read failure on the block timestamp must also not crash — the caller still gets the run
+ (if readable) and ``None`` for the timestamp."""
from skyvern.forge import app as forge_app
run = _fake_run()
@@ -255,11 +158,7 @@ async def test_read_progress_sources_swallows_progress_timestamps_errors(
)
monkeypatch.setattr(forge_app, "DATABASE", db)
- read_run, read_step_ts, read_block_ts = await _read_progress_sources(_FakeCtx(), "wr_1")
-
- assert read_run is run
- assert read_step_ts is None
- assert read_block_ts is None
+ assert await _read_progress_sources(_FakeCtx(), "wr_1") == (run, None)
# ---------------------------------------------------------------------------
@@ -336,28 +235,9 @@ async def test_fallback_page_info_bounds_a_title_that_never_resolves_and_keeps_t
assert page_title == ""
-async def test_stagnation_error_message_does_not_invite_retry() -> None:
- """The exact SKY-9163 bug: the old copy said "likely stuck repeating
- failing actions" which the LLM read as "try again". The stagnation
- message must explicitly discourage retry."""
- msg = await _watchdog_error_message(
- "stagnation", _ErrorCtx(), "wr_test", _fake_run(), RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10
- )
-
- assert "timed out" not in msg.lower()
- assert "likely stuck repeating" not in msg.lower()
- assert str(RUN_BLOCKS_STAGNATION_WINDOW_SECONDS) in msg
- assert "Run ID: wr_test" in msg
- assert "get_run_results" in msg
- assert "Do NOT re-invoke block-running tools" in msg
-
-
@pytest.mark.asyncio
async def test_ceiling_error_message_advises_splitting() -> None:
- """The ceiling path is rare (a runaway run that keeps making progress
- past 20 min). Its error must tell the LLM to split the workflow, not
- retry — a longer run won't fit either."""
+ """The error must tell the LLM to split the workflow, not retry — a longer run won't fit either."""
quiet_budget = RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10
msg = await _watchdog_error_message("ceiling", _ErrorCtx(), "wr_test", _fake_run(), quiet_budget)
@@ -366,6 +246,7 @@ async def test_ceiling_error_message_advises_splitting() -> None:
assert "split" in msg.lower()
assert "Run ID: wr_test" in msg
assert "get_run_results" in msg
+ assert "Do NOT re-invoke block-running tools" in msg
@pytest.mark.asyncio
@@ -417,7 +298,7 @@ async def test_paused_error_message_reports_a_wait_not_an_uncertain_outcome() ->
async def test_non_paused_error_messages_keep_the_run_id_for_the_model() -> None:
"""The other arms never direct a relay — they tell the model to look the run up — so stripping
the id there would take away the only handle it has."""
- exit_reasons: tuple[WatchdogExitReason, ...] = ("stagnation", "ceiling", "task_exit_unfinalized")
+ exit_reasons: tuple[WatchdogExitReason, ...] = ("ceiling", "task_exit_unfinalized")
for exit_reason in exit_reasons:
msg = await _watchdog_error_message(exit_reason, _ErrorCtx(), "wr_test", _fake_run(), 240)
@@ -433,15 +314,10 @@ async def test_non_paused_error_messages_keep_the_run_id_for_the_model() -> None
_fake_run(status="paused"),
"The run is paused, waiting for a person to approve or reject it.",
),
- (
- "stagnation",
- _fake_run(),
- f"The run stopped after no observable progress for {RUN_BLOCKS_STAGNATION_WINDOW_SECONDS}s.",
- ),
(
"ceiling",
_fake_run(),
- f"The run exceeded the {RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10}s absolute ceiling while still showing progress.",
+ f"The run did not finish within the {RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10}s absolute ceiling.",
),
(
"task_exit_unfinalized",
@@ -465,107 +341,6 @@ def test_watchdog_user_relayed_text_is_id_free_and_clears_the_output_guard(
assert_clean_user_facing_text(reason)
-# ---------------------------------------------------------------------------
-# _any_quiet_block_requested: stagnation bypass for block types that
-# legitimately do long-silent work. Without this bypass, a WAIT block with
-# wait_sec >= 90, a slow TEXT_PROMPT LLM call, or a HumanInteractionBlock
-# pausing for user input would be falsely reported as stagnation and the
-# tool would cancel a healthy run.
-# ---------------------------------------------------------------------------
-
-
-def _workflow_with_block_types(*type_value_label_pairs: tuple[str, str]) -> Any:
- """Build a minimal `last_workflow`-shaped object that
- ``_any_quiet_block_requested`` can walk. Each pair is
- ``(block_type_value, label)`` — e.g. ``("wait", "pause1")``.
- """
- blocks = [
- SimpleNamespace(label=label, block_type=SimpleNamespace(value=block_type_value))
- for block_type_value, label in type_value_label_pairs
- ]
- definition = SimpleNamespace(blocks=blocks)
- return SimpleNamespace(workflow_definition=definition)
-
-
-def test_any_quiet_block_requested_wait() -> None:
- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("wait", "pause1")))
- assert _any_quiet_block_requested(ctx, ["pause1"]) is True
-
-
-def test_any_quiet_block_requested_text_prompt() -> None:
- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("text_prompt", "prompt1")))
- assert _any_quiet_block_requested(ctx, ["prompt1"]) is True
-
-
-def test_any_quiet_block_requested_human_interaction() -> None:
- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("human_interaction", "wait_for_user")))
- assert _any_quiet_block_requested(ctx, ["wait_for_user"]) is True
-
-
-def test_any_quiet_block_requested_file_download() -> None:
- """File-download blocks can legitimately wait longer than the stagnation
- window while the browser is waiting for the download to finish."""
- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("file_download", "download_file")))
- assert _any_quiet_block_requested(ctx, ["download_file"]) is True
-
-
-def test_any_quiet_block_requested_code() -> None:
- """A code block writes its row on entry and exit and nothing between, so a login or a long
- wait inside one reads as no progress at all and the watchdog cancels a healthy run."""
- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("code", "login_and_extract")))
- assert _any_quiet_block_requested(ctx, ["login_and_extract"]) is True
-
-
-def test_any_quiet_block_requested_mixed_requested_labels_match_quiet_one() -> None:
- """When multiple blocks are requested, having any one quiet type is
- enough to disable stagnation for the whole invocation."""
- ctx = SimpleNamespace(
- last_workflow=_workflow_with_block_types(
- ("navigation", "nav1"),
- ("wait", "pause1"),
- ("extraction", "extract1"),
- )
- )
- assert _any_quiet_block_requested(ctx, ["nav1", "pause1", "extract1"]) is True
-
-
-def test_any_quiet_block_requested_only_task_blocks_returns_false() -> None:
- """The normal case: task-heavy workflows produce regular step writes.
- Stagnation is safe to enable."""
- ctx = SimpleNamespace(
- last_workflow=_workflow_with_block_types(
- ("navigation", "nav1"),
- ("extraction", "extract1"),
- )
- )
- assert _any_quiet_block_requested(ctx, ["nav1", "extract1"]) is False
-
-
-def test_any_quiet_block_requested_label_not_in_requested_ignored() -> None:
- """A WAIT block defined in the workflow but not requested in this
- invocation must not disable stagnation."""
- ctx = SimpleNamespace(
- last_workflow=_workflow_with_block_types(
- ("wait", "not_requested_pause"),
- ("navigation", "requested_nav"),
- )
- )
- assert _any_quiet_block_requested(ctx, ["requested_nav"]) is False
-
-
-def test_any_quiet_block_requested_no_workflow_returns_false() -> None:
- """Defensive: no workflow loaded → no bypass. The loop will use its
- default stagnation behavior (safe for the common case)."""
- ctx = SimpleNamespace(last_workflow=None)
- assert _any_quiet_block_requested(ctx, ["anything"]) is False
-
-
-def test_any_quiet_block_requested_empty_labels_returns_false() -> None:
- ctx = SimpleNamespace(last_workflow=_workflow_with_block_types(("wait", "pause1")))
- assert _any_quiet_block_requested(ctx, None) is False
- assert _any_quiet_block_requested(ctx, []) is False
-
-
_HUMAN_INTERACTION_WORKFLOW_YAML = """
title: human approval example
workflow_definition:
@@ -594,6 +369,17 @@ workflow_definition:
data_extraction_goal: Extract the page heading.
"""
+_NAVIGATION_WORKFLOW_YAML = """
+title: navigation example
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: navigation
+ label: open_page
+ url: https://example.com
+ navigation_goal: Open the page.
+"""
+
_CODE_WORKFLOW_YAML = """
title: code example
workflow_definition:
@@ -610,6 +396,18 @@ def _adopted_detached_tasks(before: set[Any]) -> list[Any]:
return [task for task in run_execution._DETACHED_CLEANUP_TASKS if task not in before]
+def _install_advancing_clock(monkeypatch: pytest.MonkeyPatch, step_seconds: float = 600.0) -> Callable[[], float]:
+ elapsed = 0.0
+
+ def _monotonic() -> float:
+ nonlocal elapsed
+ elapsed += step_seconds
+ return elapsed
+
+ monkeypatch.setattr(run_execution, "time", SimpleNamespace(monotonic=_monotonic))
+ return lambda: elapsed
+
+
@pytest.mark.asyncio
async def test_paused_run_is_reported_as_a_pause_and_left_running(monkeypatch: pytest.MonkeyPatch) -> None:
"""A run paused at a human_interaction block with nobody responding: the watchdog must leave
@@ -621,6 +419,7 @@ async def test_paused_run_is_reported_as_a_pause_and_left_running(monkeypatch: p
workflow_yaml=_HUMAN_INTERACTION_WORKFLOW_YAML,
polled_status="paused",
)
+ _install_advancing_clock(monkeypatch)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.staged_workflow = harness["workflow"]
ctx.frontier_resume_session_id = "pbs_run"
@@ -730,7 +529,7 @@ async def test_a_watchdog_terminated_run_still_carries_its_per_block_page_facts(
run_result_action_row("tsk_search", ActionType.CLICK, ActionStatus.completed),
],
)
- monkeypatch.setattr(run_execution, "RUN_BLOCKS_STAGNATION_WINDOW_SECONDS", 0)
+ _install_advancing_clock(monkeypatch)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.staged_workflow = harness["workflow"]
ctx.frontier_resume_session_id = "pbs_run"
@@ -740,7 +539,7 @@ async def test_a_watchdog_terminated_run_still_carries_its_per_block_page_facts(
)
data = result["data"]
- assert data["control_signal"]["kind"] == "watchdog_stagnation"
+ assert data["control_signal"]["kind"] == "watchdog_ceiling"
assert data["observed_block_end_urls"] == {
"run_search": "https://fixture.test/results/widget/page-1",
"select_first_result": "https://fixture.test/results/widget",
@@ -752,14 +551,14 @@ async def test_a_watchdog_terminated_run_still_carries_its_per_block_page_facts(
@pytest.mark.asyncio
async def test_non_paused_watchdog_exit_still_cancels_and_clears(monkeypatch: pytest.MonkeyPatch) -> None:
- """The pause carve-out is scoped to the pause: a stagnating run still gets cancelled and still
- releases the run-session association."""
+ """The pause carve-out is scoped to the pause: a run that reaches the ceiling still gets cancelled
+ and still releases the run-session association."""
harness = await _install_run_harness(
monkeypatch,
workflow_yaml=_EXTRACTION_WORKFLOW_YAML,
polled_status="running",
)
- monkeypatch.setattr(run_execution, "RUN_BLOCKS_STAGNATION_WINDOW_SECONDS", 0)
+ _install_advancing_clock(monkeypatch)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.staged_workflow = harness["workflow"]
ctx.frontier_resume_session_id = "pbs_run"
@@ -767,7 +566,7 @@ async def test_non_paused_watchdog_exit_still_cancels_and_clears(monkeypatch: py
result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
- assert result["data"]["control_signal"]["kind"] == "watchdog_stagnation"
+ assert result["data"]["control_signal"]["kind"] == "watchdog_ceiling"
harness["cancel_run_task"].assert_awaited_once()
harness["clear"].assert_awaited_once()
assert _adopted_detached_tasks(before) == []
@@ -813,8 +612,7 @@ async def test_non_success_watchdog_result_types_selected_failed_block_locators(
)
observe = AsyncMock(return_value=[{"authored_selector": "#submit", "unobserved_reason": "run_page_unavailable"}])
monkeypatch.setattr(run_execution, "_observe_authored_locators", observe)
- monkeypatch.setattr(run_execution, "RUN_BLOCKS_STAGNATION_WINDOW_SECONDS", 0)
- monkeypatch.setattr(run_execution, "_any_quiet_block_requested", lambda *_args, **_kwargs: False)
+ _install_advancing_clock(monkeypatch)
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
ctx.staged_workflow = harness["workflow"]
ctx.frontier_resume_session_id = "pbs_run"
@@ -834,10 +632,8 @@ async def test_non_success_watchdog_result_types_selected_failed_block_locators(
@pytest.mark.asyncio
[email protected]("_repeat", range(3))
async def test_progressing_worker_run_crosses_legacy_boundary_and_returns_terminal_result(
monkeypatch: pytest.MonkeyPatch,
- _repeat: int,
) -> None:
harness = await _install_run_harness(
monkeypatch,
@@ -872,11 +668,11 @@ async def test_progressing_worker_run_crosses_legacy_boundary_and_returns_termin
)
)
- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime, datetime]:
+ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime]:
nonlocal elapsed
elapsed, status = next(progress)
marker = datetime(2026, 4, 21, 12, 0, 0, tzinfo=UTC) + timedelta(seconds=elapsed)
- return _fake_run(status=status, modified_at=marker), marker, marker
+ return _fake_run(status=status, modified_at=marker), marker
monkeypatch.setattr(run_execution, "_read_progress_sources", _read_progress)
monkeypatch.setattr(run_execution, "time", SimpleNamespace(monotonic=lambda: elapsed))
@@ -908,59 +704,20 @@ async def test_progressing_worker_run_crosses_legacy_boundary_and_returns_termin
@pytest.mark.asyncio
[email protected]("_repeat", range(3))
[email protected]("layout", ["direct", "loop", "nested", "finally"])
-async def test_web_search_finishes_after_silent_period_without_watchdog_cancellation(
+ ("workflow_yaml", "label", "block_type"),
+ [
+ (_NAVIGATION_WORKFLOW_YAML, "open_page", BlockType.NAVIGATION),
+ (_EXTRACTION_WORKFLOW_YAML, "extract_heading", BlockType.EXTRACTION),
+ ],
+ ids=["navigation", "extraction"],
+)
+async def test_silent_worker_run_finishes_pending_model_call_without_cancel(
monkeypatch: pytest.MonkeyPatch,
- _repeat: int,
- layout: str,
+ workflow_yaml: str,
+ label: str,
+ block_type: BlockType,
) -> None:
- workflow_yaml = """
-title: web search example
-workflow_definition:
- parameters: []
- blocks:
- - block_type: web_search
- label: search
- query: site:example.com docs
-"""
- selected_label = "search"
- if layout in {"loop", "nested"}:
- workflow_yaml = """
-title: web search loop
-workflow_definition:
- parameters:
- - parameter_type: workflow
- key: items
- workflow_parameter_type: json
- default_value: [1]
- blocks:
- - block_type: for_loop
- label: loop
- loop_over_parameter_key: items
- loop_blocks:
- - block_type: web_search
- label: search
- query: site:example.com docs
-"""
- selected_label = "search" if layout == "nested" else "loop"
- elif layout == "finally":
- workflow_yaml = """
-title: web search cleanup
-workflow_definition:
- parameters: []
- finally_block_label: search
- blocks:
- - block_type: navigation
- label: navigate
- url: https://example.com
- navigation_goal: Open the page.
- - block_type: web_search
- label: search
- query: site:example.com docs
-"""
- selected_label = "navigate"
- output = {"query": "site:example.com docs", "results": [], "total_count": 0, "prompt_output": None}
harness = await _install_run_harness(
monkeypatch,
workflow_yaml=workflow_yaml,
@@ -968,26 +725,33 @@ workflow_definition:
dispatch_to_worker=True,
terminal_blocks=[
WorkflowRunBlock(
- label="search",
- block_type=BlockType.WEB_SEARCH,
+ label=label,
+ block_type=block_type,
status="completed",
- output=output,
workflow_run_block_id="wrb_terminal",
workflow_run_id="wr_paused",
organization_id="org-1",
created_at=datetime(2026, 4, 21, 12, 0, tzinfo=UTC),
- modified_at=datetime(2026, 4, 21, 12, 2, 30, tzinfo=UTC),
+ modified_at=datetime(2026, 4, 21, 12, 3, 3, tzinfo=UTC),
)
],
)
elapsed = 0.0
- progress = iter(((0.0, "running"), (120.0, "running"), (150.0, "completed")))
+ progress = iter(
+ (
+ (0.0, "running"),
+ (89.0, "running"),
+ (91.0, "running"),
+ (182.0, "running"),
+ (183.0, "completed"),
+ )
+ )
marker = datetime(2026, 4, 21, 12, 0, tzinfo=UTC)
- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime, datetime]:
+ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime]:
nonlocal elapsed
elapsed, status = next(progress)
- return _fake_run(status=status, modified_at=marker), marker, marker
+ return _fake_run(status=status, modified_at=marker), marker
monkeypatch.setattr(run_execution, "_read_progress_sources", _read_progress)
monkeypatch.setattr(run_execution, "time", SimpleNamespace(monotonic=lambda: elapsed))
@@ -995,15 +759,81 @@ workflow_definition:
ctx.staged_workflow = harness["workflow"]
ctx.frontier_resume_session_id = "pbs_run"
- result = await _run_blocks_and_collect_debug({"block_labels": [selected_label], "parameters": {}}, ctx)
+ result = await _run_blocks_and_collect_debug({"block_labels": [label], "parameters": {}}, ctx)
- assert elapsed == 150.0
+ harness["cooperative_cancel"].assert_not_awaited()
+ assert elapsed == 183.0
assert result["ok"] is True, result
+ assert result["data"]["workflow_run_id"] == "wr_paused"
assert result["data"]["overall_status"] == "completed"
- assert result["data"]["blocks"][0]["output"] == output
harness["worker_execute"].assert_awaited_once()
+
+
+async def test_never_terminal_worker_run_is_cancelled_at_the_safety_ceiling(monkeypatch: pytest.MonkeyPatch) -> None:
+ harness = await _install_run_harness(
+ monkeypatch,
+ workflow_yaml=_EXTRACTION_WORKFLOW_YAML,
+ polled_status="running",
+ dispatch_to_worker=True,
+ )
+ _install_advancing_clock(monkeypatch)
+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")
+ ctx.staged_workflow = harness["workflow"]
+ ctx.frontier_resume_session_id = "pbs_run"
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+ assert result["ok"] is False, result
+ assert result["data"]["control_signal"]["kind"] == "watchdog_ceiling"
+ assert f"{RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10}s" in result["data"]["user_facing_summary"]
+ harness["cooperative_cancel"].assert_awaited_once_with("wr_paused")
+
+
+async def test_terminal_row_written_before_the_ceiling_cancel_wins(monkeypatch: pytest.MonkeyPatch) -> None:
+ harness = await _install_run_harness(
+ monkeypatch,
+ workflow_yaml=_EXTRACTION_WORKFLOW_YAML,
+ polled_status="running",
+ dispatch_to_worker=True,
+ )
+ forge_app.DATABASE.workflow_runs.get_workflow_run = AsyncMock(return_value=_fake_run(status="completed"))
+ _install_advancing_clock(monkeypatch)
+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")
+ ctx.staged_workflow = harness["workflow"]
+ ctx.frontier_resume_session_id = "pbs_run"
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
harness["cooperative_cancel"].assert_not_awaited()
- harness["cancel_run_task"].assert_not_awaited()
+ assert result["ok"] is True, result
+ assert result["data"]["overall_status"] == "completed"
+
+
[email protected](("status", "ok"), [("completed", True), ("failed", False)])
+async def test_immediate_terminal_worker_result_is_returned_as_recorded(
+ monkeypatch: pytest.MonkeyPatch, status: str, ok: bool
+) -> None:
+ harness = await _install_run_harness(
+ monkeypatch,
+ workflow_yaml=_EXTRACTION_WORKFLOW_YAML,
+ polled_status=status,
+ dispatch_to_worker=True,
+ )
+ clock = _install_advancing_clock(monkeypatch, step_seconds=1.0)
+ ctx = make_copilot_ctx(browser_session_id="pbs_chat")
+ ctx.staged_workflow = harness["workflow"]
+ ctx.frontier_resume_session_id = "pbs_run"
+
+ result = await _run_blocks_and_collect_debug({"block_labels": ["extract_heading"], "parameters": {}}, ctx)
+
+ assert clock() < RUN_BLOCKS_SAFETY_CEILING_SECONDS - 10
+ harness["cooperative_cancel"].assert_not_awaited()
+ assert result["ok"] is ok, result
+ assert result["data"]["overall_status"] == status
+ assert "control_signal" not in result["data"]
@pytest.mark.asyncio
@@ -1019,14 +849,14 @@ async def test_externally_cancelled_worker_run_still_cooperatively_cancels(
polling = asyncio.Event()
reads = 0
- async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime, datetime]:
+ async def _read_progress(_ctx: CopilotContext, _run_id: str) -> tuple[Any, datetime]:
nonlocal reads
reads += 1
marker = datetime(2026, 4, 21, 12, 0, reads, tzinfo=UTC)
if reads > 1:
polling.set()
await asyncio.Event().wait()
- return _fake_run(status="running", modified_at=marker), marker, marker
+ return _fake_run(status="running", modified_at=marker), marker
monkeypatch.setattr(run_execution, "_read_progress_sources", _read_progress)
diff --git a/tests/unit/test_debug_sessions_lookup_index_migration.py b/tests/unit/test_debug_sessions_lookup_index_migration.py
new file mode 100644
index 000000000..5278264d6
--- /dev/null
+++ b/tests/unit/test_debug_sessions_lookup_index_migration.py
@@ -0,0 +1,195 @@
+import functools
+import importlib.util
+import re
+import uuid
+from collections.abc import Iterator
+from datetime import datetime, timedelta
+from pathlib import Path
+from types import ModuleType
+from typing import Any, NamedTuple
+
+import pytest
+from sqlalchemy import Engine, create_engine, event, insert, text
+from sqlalchemy.engine import make_url
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.schema import CreateTable
+
+from alembic.operations import Operations
+from alembic.runtime.migration import MigrationContext
+from skyvern.config import settings
+from skyvern.forge.sdk.db.agent_db import AgentDB
+from skyvern.forge.sdk.db.models import DebugSessionModel
+
+INDEX_NAME = "ix_debug_sessions_org_wpid_user_created_at"
+
+
+def _migration() -> ModuleType:
+ # Matched by suffix: the open-source mirror regenerates this migration under its own date and revision id, and
+ # names it after the slug of the docstring's first line, so that line must stay "index debug_sessions lookup".
+ versions = Path(__file__).resolve().parents[2] / "alembic/versions"
+ matches = sorted(versions.glob("*_index_debug_sessions_lookup.py"))
+ if not matches:
+ raise AssertionError(f"no index_debug_sessions_lookup migration found in {versions}")
+ spec = importlib.util.spec_from_file_location("index_debug_sessions_lookup_migration", matches[-1])
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+class _Schema(NamedTuple):
+ engine: Engine
+ connect_args: dict[str, str]
+
+
+def scratch_schema() -> Iterator[_Schema]:
+ url = make_url(str(settings.DATABASE_STRING))
+ if url.get_backend_name() != "postgresql":
+ pytest.skip("requires PostgreSQL")
+ schema = f"debug_sessions_index_{uuid.uuid4().hex}"
+ connect_args = {"options": f"-csearch_path={schema}"}
+ engine = create_engine(url.set(drivername="postgresql+psycopg"), connect_args=connect_args)
+ with engine.begin() as connection:
+ connection.exec_driver_sql(f'CREATE SCHEMA "{schema}"')
+ connection.execute(CreateTable(DebugSessionModel.__table__))
+ try:
+ yield _Schema(engine, connect_args)
+ finally:
+ with engine.begin() as connection:
+ connection.exec_driver_sql(f'DROP SCHEMA "{schema}" CASCADE')
+ engine.dispose()
+
+
+def _upgrade(engine: Engine) -> None:
+ with engine.connect() as connection, Operations.context(MigrationContext.configure(connection)):
+ _migration().upgrade()
+
+
+def _index_state(engine: Engine) -> tuple[bool, str] | None:
+ with engine.connect() as connection:
+ row = connection.execute(
+ text(
+ "SELECT indisvalid, pg_get_indexdef(indexrelid) FROM pg_catalog.pg_index "
+ "WHERE indexrelid = to_regclass(:name)"
+ ),
+ {"name": INDEX_NAME},
+ ).one_or_none()
+ return None if row is None else (row[0], row[1])
+
+
+def _session_row(organization_id: str, name: str, **overrides: Any) -> dict[str, Any]:
+ return {
+ "debug_session_id": f"ds_{name}",
+ "organization_id": organization_id,
+ "browser_session_id": f"pbs_{name}",
+ "workflow_permanent_id": "wpid_target",
+ "user_id": "user_target",
+ "status": "created",
+ "deleted_at": None,
+ **overrides,
+ }
+
+
+def _generic_plan(engine: Engine, statement: str, parameters: dict[str, Any]) -> str:
+ names: list[str] = []
+
+ def number(match: re.Match[str]) -> str:
+ if match.group(1) not in names:
+ names.append(match.group(1))
+ return f"${names.index(match.group(1)) + 1}"
+
+ prepared = re.sub(r"%\((\w+)\)s", number, statement)
+ arguments = ", ".join(
+ f"'{value}'" if isinstance(value, str) else str(value) for value in (parameters[name] for name in names)
+ )
+ with engine.connect() as connection:
+ connection.exec_driver_sql("SET LOCAL plan_cache_mode = force_generic_plan")
+ # A seq scan always wins on a seven-row table, so disable it to ask only whether the index can serve the query.
+ connection.exec_driver_sql("SET LOCAL enable_seqscan = off")
+ connection.exec_driver_sql(f"PREPARE debug_session_lookup AS {prepared}")
+ try:
+ plan = connection.exec_driver_sql(f"EXPLAIN EXECUTE debug_session_lookup({arguments})").scalars().all()
+ finally:
+ connection.exec_driver_sql("DEALLOCATE debug_session_lookup")
+ connection.rollback()
+ return "\n".join(plan)
+
+
+async def test_lookup_returns_the_newest_open_session_through_the_index_under_a_generic_plan(
+ scratch_schema: _Schema,
+) -> None:
+ schema_engine = scratch_schema.engine
+ _upgrade(schema_engine)
+ organization_id = "o_debug_lookup"
+ started = datetime(2026, 9, 1)
+ with schema_engine.begin() as connection:
+ connection.execute(
+ insert(DebugSessionModel),
+ [
+ _session_row(organization_id, "older_open", created_at=started),
+ _session_row(organization_id, "newest_open", created_at=started + timedelta(minutes=1)),
+ _session_row(
+ organization_id, "completed", created_at=started + timedelta(minutes=2), status="completed"
+ ),
+ _session_row(organization_id, "deleted", created_at=started + timedelta(minutes=3), deleted_at=started),
+ _session_row(organization_id, "other_user", created_at=started + timedelta(minutes=4), user_id="u2"),
+ _session_row(
+ organization_id,
+ "other_workflow",
+ created_at=started + timedelta(minutes=5),
+ workflow_permanent_id="w2",
+ ),
+ _session_row("o_other", "other_org", created_at=started + timedelta(minutes=6)),
+ ],
+ )
+
+ async_engine = create_async_engine(schema_engine.url, connect_args=scratch_schema.connect_args)
+ statements: list[tuple[str, dict[str, Any]]] = []
+
+ def capture(_conn: Any, _cursor: Any, statement: str, parameters: Any, *_args: Any) -> None:
+ if "FROM debug_sessions" in statement:
+ statements.append((statement, parameters))
+
+ event.listen(async_engine.sync_engine, "before_cursor_execute", capture)
+ try:
+ found = await AgentDB(str(async_engine.url), db_engine=async_engine).debug.get_debug_session(
+ organization_id=organization_id,
+ user_id="user_target",
+ workflow_permanent_id="wpid_target",
+ )
+ finally:
+ event.remove(async_engine.sync_engine, "before_cursor_execute", capture)
+ await async_engine.dispose()
+
+ assert found is not None and found.debug_session_id == "ds_newest_open"
+ [(statement, parameters)] = statements
+ # psycopg prepares a statement after five runs on a connection, and Postgres may then plan it generically,
+ # with status as an unknown parameter.
+ plan = _generic_plan(schema_engine, statement, parameters)
+ assert f"Index Scan Backward using {INDEX_NAME}" in plan, plan
+ assert "Sort" not in plan, plan
+
+
+def test_upgrade_rebuilds_an_invalid_index_left_by_a_failed_concurrent_build(scratch_schema: _Schema) -> None:
+ schema_engine = scratch_schema.engine
+ with schema_engine.begin() as connection:
+ connection.execute(insert(DebugSessionModel), [_session_row("o_debug_lookup", "only")])
+ with schema_engine.connect().execution_options(isolation_level="AUTOCOMMIT") as connection:
+ with pytest.raises(DBAPIError, match="division by zero"):
+ connection.exec_driver_sql(
+ f"CREATE INDEX CONCURRENTLY {INDEX_NAME} ON debug_sessions ((length(user_id) / 0))"
+ )
+ leftover = _index_state(schema_engine)
+ assert leftover is not None and leftover[0] is False
+
+ _upgrade(schema_engine)
+
+ state = _index_state(schema_engine)
+ assert state is not None
+ valid, definition = state
+ assert valid is True
+ assert "(organization_id, workflow_permanent_id, user_id, created_at)" in definition
diff --git a/tests/unit/test_request_logging.py b/tests/unit/test_request_logging.py
index 6b448dc52..15d671aa0 100644
--- a/tests/unit/test_request_logging.py
+++ b/tests/unit/test_request_logging.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import io
import json
-import logging
import typing
from datetime import UTC, datetime
from types import SimpleNamespace
@@ -10,7 +9,6 @@ from unittest.mock import AsyncMock, MagicMock
import psycopg.errors
import pytest
-import structlog
from fastapi import FastAPI, HTTPException, Response
from fastapi.responses import JSONResponse, StreamingResponse
from fastapi.testclient import TestClient
@@ -18,7 +16,7 @@ from sqlalchemy.exc import OperationalError
from starlette.requests import ClientDisconnect, Request
from starlette.types import ASGIApp, Receive, Scope, Send
-from skyvern.forge import api_app, forge_app_initializer, request_logging
+from skyvern.forge import api_app, request_logging
from skyvern.forge.log_redaction import (
REDACTED,
SENSITIVE_FIELDS,
@@ -39,7 +37,6 @@ from skyvern.forge.request_logging import (
log_raw_request_middleware,
set_request_organization,
)
-from skyvern.forge.sdk.forge_log import setup_logger
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.services.org_auth_service import apply_request_org_context
@@ -129,6 +126,11 @@ class TestIsSensitiveKey:
"client_secret",
"access_token",
"refresh_token",
+ # Payment-provider payer details and webhook signatures
+ "billing_details",
+ "payment_method_details",
+ "receipt_email",
+ "Stripe-Signature",
],
)
def test_sensitive_keys_are_redacted(self, key: str) -> None:
@@ -1163,27 +1165,6 @@ _ROUTE_CALL_DEPTH = 25
_TEST_ORGANIZATION_ID = "o_385835488455492960"
-def rendered_log_stream(monkeypatch: pytest.MonkeyPatch) -> typing.Iterator[io.StringIO]:
- monkeypatch.setattr(request_logging.settings, "LOG_RAW_API_REQUESTS", True)
- monkeypatch.setattr(request_logging.settings, "JSON_LOGGING", True)
- root_logger = logging.getLogger()
- saved_handlers = root_logger.handlers[:]
- saved_structlog_config = structlog.get_config()
- setup_logger()
- # create_api_app configures logging once per process; keep it from replacing this handler.
- monkeypatch.setattr(forge_app_initializer, "_SERVER_LOGGING_CONFIGURED", True)
- stream = io.StringIO()
- handler = root_logger.handlers[0]
- assert isinstance(handler, logging.StreamHandler)
- handler.setStream(stream)
- try:
- yield stream
- finally:
- root_logger.handlers[:] = saved_handlers
- structlog.configure(**saved_structlog_config)
-
-
def _raise_from_depth(depth: int) -> None:
if depth == 0:
raise RuntimeError("connection pool exhausted")
diff --git a/tests/unit/test_saved_session_proxy_pin.py b/tests/unit/test_saved_session_proxy_pin.py
index 5d3b1cbbe..370dd5e16 100644
--- a/tests/unit/test_saved_session_proxy_pin.py
+++ b/tests/unit/test_saved_session_proxy_pin.py
@@ -11,7 +11,12 @@ from skyvern.forge.sdk.schemas.browser_profiles import BrowserProfile
from skyvern.forge.sdk.schemas.persistent_browser_sessions import FORCED_WORKFLOW_SESSION_RUNNABLE_TYPE
from skyvern.forge.sdk.workflow.browser_profile_key import build_browser_profile_key_digest
from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter, WorkflowParameter, WorkflowParameterType
-from skyvern.forge.sdk.workflow.models.workflow import WorkflowRequestBody, WorkflowRun
+from skyvern.forge.sdk.workflow.models.workflow import (
+ WorkflowDefinition,
+ WorkflowRequestBody,
+ WorkflowRun,
+ workflow_definition_sha256,
+)
from skyvern.forge.sdk.workflow.service import WorkflowService
from skyvern.schemas.browser_session_kind import BrowserSessionKind
from skyvern.schemas.proxy_pinning import derive_proxy_session_id, is_proxy_session_id
@@ -46,7 +51,7 @@ def _workflow(
code_version=None,
adaptive_caching=False,
sequential_key=None,
- workflow_definition=SimpleNamespace(parameters=parameters or [], blocks=[], retry_policy=None),
+ workflow_definition=WorkflowDefinition(parameters=parameters or [], blocks=[]),
)
@@ -792,8 +797,11 @@ async def test_create_workflow_run_non_force_path_single_create_no_update(monkey
)
monkeypatch.setattr(app.DATABASE.workflow_runs, "create_workflow_run", create_workflow_run)
monkeypatch.setattr(app.DATABASE.workflow_runs, "update_workflow_run", update_workflow_run)
+ workflow = _workflow()
+ service = WorkflowService()
+ monkeypatch.setattr(service, "get_workflow", AsyncMock(return_value=workflow))
- result = await WorkflowService().create_workflow_run(
+ result = await service.create_workflow_run(
workflow_request=request,
workflow_permanent_id="wpid_test",
workflow_id="wf_test",
@@ -833,6 +841,7 @@ async def test_create_workflow_run_non_force_path_single_create_no_update(monkey
ignore_inherited_workflow_system_prompt=False,
copilot_session_id=None,
created_by=None,
+ workflow_definition_sha256=workflow_definition_sha256(workflow.workflow_definition),
)
update_workflow_run.assert_not_awaited()
@@ -1200,3 +1209,13 @@ async def test_create_persistent_browser_session_does_not_inherit_profile_pin_by
assert session.proxy_location is None
assert session.proxy_session_id is None
assert session.browser_profile_id == "bp_managed"
+
+
+async def test_a_created_run_records_the_digest_of_the_definition_it_runs(monkeypatch: pytest.MonkeyPatch) -> None:
+ workflow = _workflow(parameters=[_workflow_parameter("credential_id", default_value="cred_default")])
+
+ created = await _create_forced_workflow_run(monkeypatch, workflow=workflow)
+
+ recorded = created.create_workflow_run.await_args.kwargs["workflow_definition_sha256"]
+ assert recorded == workflow_definition_sha256(workflow.workflow_definition)
diff --git a/tests/unit/test_task_v2_completion_gate.py b/tests/unit/test_task_v2_completion_gate.py
index d2f5af296..77b56fc4e 100644
--- a/tests/unit/test_task_v2_completion_gate.py
+++ b/tests/unit/test_task_v2_completion_gate.py
@@ -15,7 +15,7 @@ import pytest
from skyvern.config import Settings
from skyvern.forge.sdk.prompting import PromptEngine
from skyvern.forge.sdk.schemas.task_v2 import TaskV2Status
-from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition, WorkflowRunStatus
from skyvern.forge.sdk.workflow.service import WorkflowService
from skyvern.services import planner_levers, task_v2_service
from skyvern.services.task_v2_service import _should_run_post_block_completion_check
@@ -39,7 +39,12 @@ async def _run_task_v2_wiring_case(
task_v2.url, task_v2.workflow_run_id = "https://example.test", "wr_test"
queued_run = MagicMock(workflow_run_id="wr_test", workflow_id="wf_test", status=WorkflowRunStatus.queued)
running_run = MagicMock(workflow_run_id="wr_test", workflow_id="wf_test", status=WorkflowRunStatus.running)
- workflow = MagicMock(workflow_id="wf_test", workflow_permanent_id="wp_test", title="Test")
+ workflow = MagicMock(
+ workflow_id="wf_test",
+ workflow_permanent_id="wp_test",
+ title="Test",
+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),
+ )
block_result = MagicMock(failure_reason=None, output_parameter_value={}, success=completion_case)
block = SimpleNamespace(execute_safe=AsyncMock(return_value=block_result))
scraped_page = SimpleNamespace(screenshots=[])
@@ -220,6 +225,7 @@ async def test_first_iteration_goto_url_has_no_completion_criterion(
status="published",
extra_http_headers=None,
cdp_connect_headers=None,
+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),
)
block_result = SimpleNamespace(
status="failed",
@@ -352,6 +358,7 @@ async def test_missing_task_type_failure_finalizes_with_organization_scope(
title="Test workflow",
description=None,
status="published",
+ workflow_definition=WorkflowDefinition(parameters=[], blocks=[]),
)
page = MagicMock()
scraped_page = SimpleNamespace(screenshots=[])
diff --git a/tests/unit/test_workflow_block_engine.py b/tests/unit/test_workflow_block_engine.py
index c4baf7ff5..3979bea60 100644
--- a/tests/unit/test_workflow_block_engine.py
+++ b/tests/unit/test_workflow_block_engine.py
@@ -542,9 +542,7 @@ async def _persisted_engine_from_execute_workflow_blocks(
block = block or _make_block(TaskBlock, label="e2e_block")
workflow = MagicMock()
- workflow.workflow_definition.blocks = [block]
- workflow.workflow_definition.version = 1
- workflow.workflow_definition.finally_block_label = None
+ workflow.workflow_definition = WorkflowDefinition(parameters=[], blocks=[block])
workflow.status = workflow_status
workflow_run = MagicMock()
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 0c1b00c26..9afa29f62 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -101,6 +101,7 @@ from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatRequest,
WorkflowCopilotChatSender,
WorkflowCopilotClearProposedWorkflowRequest,
+ WorkflowCopilotCredentialRegistration,
WorkflowCopilotDisableAutoAcceptRequest,
WorkflowCopilotMessageFeedbackRequest,
WorkflowCopilotStreamErrorUpdate,
@@ -4557,7 +4558,10 @@ async def test_request_webhook_is_validated_before_staging_once(
await captured["handler"](copilot_stream)
frames = [call.args[0].model_dump(mode="json") for call in copilot_stream.send.await_args_list]
- if webhook_kind == "blocked":
+ # The Test action runs the proposal under its own settings, so the editor's webhook is never carried.
+ carried = product_action is None
+ expected_webhook = private_url if carried or webhook_kind == "stored" else None
+ if webhook_kind == "blocked" and carried:
assert not contexts, "The agent must not receive a request with a newly carried blocked webhook"
errors = [frame for frame in frames if frame["type"] == "error"]
assert errors and "webhook_callback_url" in errors[0]["error"]
@@ -4566,7 +4570,7 @@ async def test_request_webhook_is_validated_before_staging_once(
else:
assert contexts and all(update["ok"] for update in updates), (updates, frames)
ctx = contexts[0]
- assert ctx.staged_workflow.webhook_callback_url == private_url
+ assert ctx.staged_workflow.webhook_callback_url == expected_webhook
assert private_url not in ctx.workflow_yaml
chat.proposed_workflow = workflow_copilot_route._build_proposed_workflow_data(
ctx.staged_workflow,
@@ -4584,8 +4588,8 @@ async def test_request_webhook_is_validated_before_staging_once(
await workflow_copilot_chat_post(api_key_request, request, organization)
await captured["handler"](copilot_stream)
assert len(contexts) == 2 and all(update["ok"] for update in updates), updates
- assert contexts[-1].staged_workflow.webhook_callback_url == private_url
- assert validated_urls == ([] if webhook_kind == "stored" else [private_url])
+ assert contexts[-1].staged_workflow.webhook_callback_url == expected_webhook
+ assert validated_urls == ([private_url] if carried and webhook_kind != "stored" else [])
@pytest.mark.asyncio
@@ -6091,7 +6095,7 @@ async def test_reconcile_leaves_a_turn_that_just_resumed_from_a_credential_pause
@pytest.mark.asyncio
[email protected]("answer", ["connected", "skip", "signed_in"])
[email protected]("answer", ["connected", "skip", "signed_in", "generated"])
async def test_reconcile_between_a_card_answer_and_the_waiter_leaves_the_turn(
sqlite_engine: AsyncEngine, monkeypatch: pytest.MonkeyPatch, answer: str
) -> None:
@@ -6105,11 +6109,23 @@ async def test_reconcile_between_a_card_answer_and_the_waiter_leaves_the_turn(
sign_in = credential_pause.ManualSignIn(
browser_session_id="pbs_debug", login_urls=["https://portal.example.com/login"], profile_name="Sign-in"
)
+ registration = (
+ WorkflowCopilotCredentialRegistration(username="[email protected]", credential_name="Portal")
+ if answer == "generated"
+ else None
+ )
cache.store[credential_pause.credential_pause_active_key("org", chat_id, "turn-a")] = (
- credential_pause._encode_active_pause(token, datetime.now(UTC) + timedelta(minutes=5), manual_sign_in=sign_in)
+ credential_pause._encode_active_pause(
+ token, datetime.now(UTC) + timedelta(minutes=5), manual_sign_in=sign_in, registration=registration
+ )
)
ids = {"organization_id": "org", "workflow_copilot_chat_id": chat_id, "turn_id": "turn-a"}
- if answer == "signed_in":
+ if answer == "generated":
+ generation = await credential_pause.claim_credential_generation(cache, resume_token=token, **ids)
+ assert await credential_pause.finish_credential_generation(
+ cache, claim=generation, credential_id="cred_generated", **ids
+ )
+ elif answer == "signed_in":
claim = await credential_pause.claim_manual_sign_in(cache, resume_token=token, **ids)
profile = credential_pause.SignedInProfile(
browser_profile_id="bp_signed_in", profile_name="Sign-in", site="portal.example.com", cookie_count=1
@@ -7020,7 +7036,8 @@ async def test_test_end_to_end_route_hands_the_proposal_bound_account_to_the_age
for key in ("extra_http_headers", "cdp_connect_headers", "totp_identifier", "totp_verification_url")
}
expected_settings["totp_identifier"] = "synthetic-unsaved-identifier"
- assert private_settings == expected_settings
+ # The Test action runs the proposal under its own settings, so the editor's unsaved ones never reach the turn.
+ assert private_settings == ({} if product_action else expected_settings)
expected_yaml.pop("totp_identifier")
expected_yaml.pop("totp_verification_url")
expected_yaml["extra_http_headers"] = {"Authorization": "***"}
diff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.py
index a5267ff10..64e6dd658 100644
--- a/tests/unit/workflow/test_code_block_recorder.py
+++ b/tests/unit/workflow/test_code_block_recorder.py
@@ -190,6 +190,9 @@ class FakePage:
def set_default_timeout(self, timeout: float) -> None:
self.default_timeout = timeout
+ def is_closed(self) -> bool:
+ return False
+
async def goto(self, url, **kwargs): # noqa: ANN001, ANN003, ANN201
return None