Skip to content

Automate browser based workflows with AI

feat(SKY-12153): fallback credentials with automatic run retry (#7521)

Aaron Perez27c90a1df433
diff --git a/alembic/versions/2026_07_17_1752-d9ab2c4d41e5_add_credential_fallback_fields.py b/alembic/versions/2026_07_17_1752-d9ab2c4d41e5_add_credential_fallback_fields.py
new file mode 100644
index 000000000..0a129e307
--- /dev/null
+++ b/alembic/versions/2026_07_17_1752-d9ab2c4d41e5_add_credential_fallback_fields.py
@@ -0,0 +1,56 @@
+"""add credential fallback fields
+
+Revision ID: d9ab2c4d41e5
+Revises: 1915b0e1126e
+Create Date: 2026-07-17T17:52:48.243761+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "d9ab2c4d41e5"
+down_revision: Union[str, None] = "1915b0e1126e"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ op.execute("SET LOCAL lock_timeout = '5s'")
+ op.add_column("credential_parameters", sa.Column("fallback_credential_ids", sa.JSON(), nullable=True))
+ op.add_column("credential_parameters", sa.Column("fallback_trigger", sa.String(), nullable=True))
+ op.add_column("workflow_runs", sa.Column("retried_from_workflow_run_id", sa.String(), nullable=True))
+ op.add_column("workflow_runs", sa.Column("fallback_attempt", sa.Integer(), nullable=True))
+
+ with op.get_context().autocommit_block():
+ op.execute("SET lock_timeout = '5s'")
+ op.execute("SET statement_timeout = '1h'")
+ op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_workflow_runs_retried_from_workflow_run_id")
+ op.execute(
+ """
+ CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS ix_workflow_runs_retried_from_workflow_run_id
+ ON workflow_runs (retried_from_workflow_run_id)
+ WHERE retried_from_workflow_run_id IS NOT NULL
+ """
+ )
+ op.execute("RESET statement_timeout")
+ op.execute("RESET lock_timeout")
+
+
+def downgrade() -> None:
+ with op.get_context().autocommit_block():
+ op.execute("SET lock_timeout = '5s'")
+ op.execute("SET statement_timeout = '1h'")
+ op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_workflow_runs_retried_from_workflow_run_id")
+ op.execute("RESET statement_timeout")
+ op.execute("RESET lock_timeout")
+
+ op.execute("SET LOCAL lock_timeout = '5s'")
+ op.drop_column("workflow_runs", "fallback_attempt")
+ op.drop_column("workflow_runs", "retried_from_workflow_run_id")
+ op.drop_column("credential_parameters", "fallback_trigger")
+ op.drop_column("credential_parameters", "fallback_credential_ids")
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.ts
index 49aee7a15..b7e6ed3ce 100644
--- a/skyvern-frontend/src/api/types.ts
+++ b/skyvern-frontend/src/api/types.ts
@@ -744,6 +744,7 @@ export type WorkflowRunApiResponse = {
workflow_permanent_id: string;
workflow_run_id: string;
workflow_title: string | null;
+ retried_from_workflow_run_id?: string | null;
};
export const TaskRunType = {
@@ -810,6 +811,8 @@ export type WorkflowRunStatusApiResponse = {
waiting_for_verification_code?: boolean;
verification_code_identifier?: string | null;
verification_code_polling_started_at?: string | null;
+ retried_from_workflow_run_id?: string | null;
+ retried_by_workflow_run_id?: string | null;
};
export type WorkflowRunStatusApiResponseWithWorkflow = {
@@ -849,6 +852,8 @@ export type WorkflowRunStatusApiResponseWithWorkflow = {
waiting_for_verification_code?: boolean;
verification_code_identifier?: string | null;
verification_code_polling_started_at?: string | null;
+ retried_from_workflow_run_id?: string | null;
+ retried_by_workflow_run_id?: string | null;
};
export type TaskGenerationApiResponse = {
diff --git a/skyvern-frontend/src/components/CredentialFallbackRetryBadge.tsx b/skyvern-frontend/src/components/CredentialFallbackRetryBadge.tsx
new file mode 100644
index 000000000..a720ee03b
--- /dev/null
+++ b/skyvern-frontend/src/components/CredentialFallbackRetryBadge.tsx
@@ -0,0 +1,31 @@
+import { UpdateIcon } from "@radix-ui/react-icons";
+
+import { Tip } from "@/components/Tip";
+
+type Props = {
+ retriedFromWorkflowRunId: string | null | undefined;
+};
+
+const retryConfig = {
+ icon: <UpdateIcon className="size-3.5 text-blue-400" />,
+ label: "Automatic retry with fallback credential",
+};
+
+function CredentialFallbackRetryBadge({ retriedFromWorkflowRunId }: Props) {
+ if (!retriedFromWorkflowRunId) {
+ return null;
+ }
+
+ return (
+ <Tip content={retryConfig.label}>
+ <span
+ aria-label={retryConfig.label}
+ className="inline-flex shrink-0 items-center"
+ >
+ {retryConfig.icon}
+ </span>
+ </Tip>
+ );
+}
+
+export { CredentialFallbackRetryBadge };
diff --git a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
index 9dfc2396a..cd8a321d8 100644
--- a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
+++ b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx
@@ -15,6 +15,7 @@ import { ProxyLocation } from "@/api/types";
import { ProxySelector } from "@/components/ProxySelector";
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
import { Button } from "@/components/ui/button";
+import { Badge } from "@/components/ui/badge";
import {
Accordion,
AccordionContent,
@@ -59,6 +60,7 @@ import { parseHeaderJson } from "@/util/secretHeaders";
import { MAX_SCREENSHOT_SCROLLS_DEFAULT } from "./editor/nodes/Taskv2Node/types";
import { getLabelForWorkflowParameterType } from "./editor/workflowEditorUtils";
import {
+ CredentialFallbackTrigger,
CredentialParameter,
WorkflowApiResponse,
WorkflowBlock,
@@ -74,7 +76,10 @@ import {
parseJsonWorkflowParameterValue,
validateJsonWorkflowParameterValue,
} from "./utils";
-import { getLoginCredentialInputs } from "./runWorkflowCredentials";
+import {
+ getLoginCredentialInputs,
+ getRotatingCredentialIds,
+} from "./runWorkflowCredentials";
import { useCredentialsQuery } from "./hooks/useCredentialsQuery";
import { visitWorkflowBlocks } from "./workflowBlockUtils";
@@ -354,6 +359,47 @@ function getLoginCredentialDisplayText(
};
}
+function FallbackCredentialList({
+ fallbackCredentialIds,
+ fallbackTrigger,
+ credentialNamesById,
+}: {
+ fallbackCredentialIds: Array<string>;
+ fallbackTrigger: CredentialFallbackTrigger | null;
+ credentialNamesById: Map<string, string>;
+}) {
+ if (fallbackCredentialIds.length === 0) {
+ return null;
+ }
+
+ const triggerText =
+ fallbackTrigger === "any_failure"
+ ? "for any reason"
+ : "on credential failures";
+
+ return (
+ <div className="space-y-2">
+ <ol className="space-y-1">
+ {fallbackCredentialIds.map((credentialId, index) => (
+ <li
+ key={credentialId}
+ className="flex min-w-0 items-center gap-2 rounded border border-slate-700/60 bg-slate-950/40 px-2 py-1.5 text-xs text-slate-200"
+ >
+ <span className="shrink-0 text-slate-400">{index + 1}.</span>
+ <span className="truncate">
+ {credentialNamesById.get(credentialId) ?? credentialId}
+ </span>
+ </li>
+ ))}
+ </ol>
+ <p className="text-xs text-slate-400">
+ If the run fails {triggerText}, Skyvern retries it automatically with
+ the next fallback.
+ </p>
+ </div>
+ );
+}
+
type RunWorkflowFormType = Record<string, unknown> & {
webhookCallbackUrl: string;
proxyLocation: ProxyLocation;
@@ -764,11 +810,18 @@ function RunWorkflowForm({
<header>
<h1 className="text-lg">Login credentials</h1>
</header>
- {loginCredentialInputs.map(({ parameter, loginBlockLabels }) => {
+ {loginCredentialInputs.map((input) => {
+ const {
+ parameter,
+ loginBlockLabels,
+ fallbackCredentialIds,
+ fallbackTrigger,
+ } = input;
const { title, description } = getLoginCredentialDisplayText(
parameter,
loginBlockLabels,
);
+ const hasFallbacks = fallbackCredentialIds.length > 0;
if (
parameter.parameter_type === WorkflowParameterTypes.Workflow
@@ -829,6 +882,47 @@ function RunWorkflowForm({
);
}
+ const credentialIds = getRotatingCredentialIds(parameter);
+
+ if (credentialIds.length <= 1) {
+ const primaryCredentialId =
+ credentialIds[0] ?? parameter.credential_id;
+ return (
+ <div key={parameter.key} className="flex gap-16">
+ <div className="w-72 shrink-0 text-slate-50">
+ <div className="flex items-center gap-2 text-lg">
+ {title}
+ <span className="text-sm text-slate-400">
+ credential
+ </span>
+ </div>
+ <h2 className="text-sm text-slate-400">{description}</h2>
+ </div>
+ <div className="w-full space-y-2">
+ <div className="flex min-w-0 items-center gap-2 rounded border border-slate-700/60 bg-slate-950/40 px-2 py-1.5 text-xs text-slate-200">
+ <span className="min-w-0 truncate">
+ {credentialNamesById.get(primaryCredentialId) ??
+ primaryCredentialId}
+ </span>
+ {hasFallbacks && (
+ <Badge
+ variant="outline"
+ className="shrink-0 text-xs font-normal"
+ >
+ Primary
+ </Badge>
+ )}
+ </div>
+ <FallbackCredentialList
+ fallbackCredentialIds={fallbackCredentialIds}
+ fallbackTrigger={fallbackTrigger}
+ credentialNamesById={credentialNamesById}
+ />
+ </div>
+ </div>
+ );
+ }
+
return (
<FormField
key={parameter.key}
@@ -845,6 +939,14 @@ function RunWorkflowForm({
credentialNamesById={credentialNamesById}
title={title}
description={description}
+ showPrimaryBadge={hasFallbacks}
+ fallbackContent={
+ <FallbackCredentialList
+ fallbackCredentialIds={fallbackCredentialIds}
+ fallbackTrigger={fallbackTrigger}
+ credentialNamesById={credentialNamesById}
+ />
+ }
/>
)}
/>
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx
index d24c2d25a..23be1e093 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx
@@ -1,14 +1,16 @@
// @vitest-environment jsdom
-import { cleanup, render, screen } from "@testing-library/react";
+import { cleanup, render, screen, within } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { MemoryRouter, Route, Routes, useLocation } from "react-router-dom";
+import { Status, type WorkflowRunApiResponse } from "@/api/types";
import CloudContext from "@/store/CloudContext";
import { PageSlotsProvider, type PageSlots } from "@/store/PageSlots";
import { WorkflowPage } from "./WorkflowPage";
-const { mockFeatureFlagEnabled } = vi.hoisted(() => ({
+const { mockFeatureFlagEnabled, mockWorkflowRunsQuery } = vi.hoisted(() => ({
mockFeatureFlagEnabled: vi.fn(),
+ mockWorkflowRunsQuery: vi.fn(),
}));
vi.mock("posthog-js/react", () => ({
@@ -63,10 +65,7 @@ vi.mock("./hooks/useWorkflowQuery", () => ({
}));
vi.mock("./hooks/useWorkflowRunsQuery", () => ({
- useWorkflowRunsQuery: () => ({
- data: [],
- isLoading: false,
- }),
+ useWorkflowRunsQuery: () => mockWorkflowRunsQuery(),
}));
vi.mock("./hooks/useWorkflowTagsBatchQuery", () => ({
@@ -131,6 +130,7 @@ type RenderOptions = {
analyticsFlagEnabled?: boolean;
pageSlots?: PageSlots;
initialEntries?: Array<string>;
+ workflowRuns?: Array<WorkflowRunApiResponse>;
};
function LocationProbe() {
@@ -143,8 +143,13 @@ function renderWorkflowPage({
analyticsFlagEnabled = true,
pageSlots = {},
initialEntries = ["/workflows/wpid_abc123"],
+ workflowRuns = [],
}: RenderOptions = {}) {
mockFeatureFlagEnabled.mockReturnValue(analyticsFlagEnabled);
+ mockWorkflowRunsQuery.mockReturnValue({
+ data: workflowRuns,
+ isLoading: false,
+ });
return render(
<CloudContext.Provider value={isCloud}>
@@ -167,6 +172,28 @@ function renderWorkflowPage({
);
}
+function makeWorkflowRun(
+ overrides: Partial<WorkflowRunApiResponse>,
+): WorkflowRunApiResponse {
+ return {
+ created_at: "2026-06-01T12:00:00.000Z",
+ failure_reason: null,
+ started_at: "2026-06-01T12:00:01.000Z",
+ finished_at: "2026-06-01T12:00:10.000Z",
+ modified_at: "2026-06-01T12:00:10.000Z",
+ proxy_location: null,
+ script_run: false,
+ status: Status.Completed,
+ webhook_callback_url: "",
+ workflow_id: "wf_abc123",
+ workflow_permanent_id: "wpid_abc123",
+ workflow_run_id: "wr_default",
+ workflow_title: "Test Workflow",
+ retried_from_workflow_run_id: null,
+ ...overrides,
+ };
+}
+
describe("WorkflowPage analytics button", () => {
it("shows the Analytics link for cloud users when the dashboard flag is enabled", () => {
renderWorkflowPage();
@@ -230,4 +257,32 @@ describe("WorkflowPage analytics button", () => {
expect(search).toContain("from=2026-06-01");
expect(search).toContain("to=2026-06-03");
});
+
+ it("shows the fallback retry badge only for runs retried from another run", () => {
+ renderWorkflowPage({
+ workflowRuns: [
+ makeWorkflowRun({
+ workflow_run_id: "wr_retry",
+ retried_from_workflow_run_id: "wr_original",
+ }),
+ makeWorkflowRun({ workflow_run_id: "wr_original" }),
+ ],
+ });
+
+ const retryRow = screen.getByText("wr_retry").closest("tr");
+ const originalRow = screen.getByText("wr_original").closest("tr");
+
+ expect(retryRow).not.toBeNull();
+ expect(originalRow).not.toBeNull();
+ expect(
+ within(retryRow!).getByLabelText(
+ "Automatic retry with fallback credential",
+ ),
+ ).not.toBeNull();
+ expect(
+ within(originalRow!).queryByLabelText(
+ "Automatic retry with fallback credential",
+ ),
+ ).toBeNull();
+ });
});
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx
index 8f1dffeb1..4dfe8317d 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx
@@ -10,6 +10,7 @@ import {
import { Tip } from "@/components/Tip";
import { Status, WorkflowRunStatusApiResponse } from "@/api/types";
import { StatusBadge } from "@/components/StatusBadge";
+import { CredentialFallbackRetryBadge } from "@/components/CredentialFallbackRetryBadge";
import { StatusFilterDropdown } from "@/components/StatusFilterDropdown";
import { Button } from "@/components/ui/button";
import {
@@ -490,7 +491,14 @@ function WorkflowPage() {
</div>
</TableCell>
<TableCell>
- <StatusBadge status={workflowRun.status} />
+ <div className="flex items-center gap-2">
+ <StatusBadge status={workflowRun.status} />
+ <CredentialFallbackRetryBadge
+ retriedFromWorkflowRunId={
+ workflowRun.retried_from_workflow_run_id
+ }
+ />
+ </div>
</TableCell>
<TableCell
className="text-muted-foreground"
diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
index c72cda03e..78ba90efb 100644
--- a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
+++ b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx
@@ -4,6 +4,7 @@ import { getClient } from "@/api/AxiosClient";
import { ProxyLocation, Status } from "@/api/types";
import { FailureCategoryBadge } from "@/components/FailureCategoryBadge";
import { StatusBadge } from "@/components/StatusBadge";
+import { CredentialFallbackRetryBadge } from "@/components/CredentialFallbackRetryBadge";
import {
SwitchBarNavigation,
type SwitchBarNavigationOption,
@@ -312,6 +313,8 @@ function WorkflowRun() {
workflowRun?.status === Status.Terminated
? "Termination Reason"
: "Failure Reason";
+ const siblingRunLinkClassName =
+ "font-mono text-sm text-neutral-600 hover:text-neutral-950 hover:underline hover:underline-offset-2 dark:text-slate-400 dark:hover:text-slate-200";
const finallyBlockInTimeline = finallyBlockLabel
? workflowRunTimeline?.find(
@@ -563,10 +566,14 @@ function WorkflowRun() {
{workflowRunIsLoading ? (
<Skeleton className="h-8 w-28" />
) : workflowRun ? (
- <StatusBadge
- className="mt-[0.27rem]"
- status={workflowRun?.status}
- />
+ <div className="mt-[0.27rem] flex items-center gap-2">
+ <StatusBadge status={workflowRun?.status} />
+ <CredentialFallbackRetryBadge
+ retriedFromWorkflowRunId={
+ workflowRun.retried_from_workflow_run_id
+ }
+ />
+ </div>
) : null}
</div>
<h2 className="text-2xl text-neutral-600 dark:text-slate-400">
@@ -575,6 +582,26 @@ function WorkflowRun() {
{workflowRunId ? (
<RunTagsEditor workflowRunId={workflowRunId} />
) : null}
+ {workflowRun && workflowPermanentId && (
+ <div className="flex flex-wrap gap-x-4 gap-y-1">
+ {workflowRun.retried_from_workflow_run_id && (
+ <Link
+ className={siblingRunLinkClassName}
+ to={`/agents/${workflowPermanentId}/${workflowRun.retried_from_workflow_run_id}/overview`}
+ >
+ Retried from {workflowRun.retried_from_workflow_run_id}
+ </Link>
+ )}
+ {workflowRun.retried_by_workflow_run_id && (
+ <Link
+ className={siblingRunLinkClassName}
+ to={`/agents/${workflowPermanentId}/${workflowRun.retried_by_workflow_run_id}/overview`}
+ >
+ Retried by {workflowRun.retried_by_workflow_run_id}
+ </Link>
+ )}
+ </div>
+ )}
{workflowRun &&
(workflowRun.started_at ||
workflowRun.finished_at ||
diff --git a/skyvern-frontend/src/routes/workflows/components/RotatingCredentialField.tsx b/skyvern-frontend/src/routes/workflows/components/RotatingCredentialField.tsx
index 1b229cd2c..d2df8996c 100644
--- a/skyvern-frontend/src/routes/workflows/components/RotatingCredentialField.tsx
+++ b/skyvern-frontend/src/routes/workflows/components/RotatingCredentialField.tsx
@@ -1,3 +1,5 @@
+import { type ReactNode } from "react";
+
import { Badge } from "@/components/ui/badge";
import { FormControl, FormItem, FormLabel } from "@/components/ui/form";
import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";
@@ -19,6 +21,8 @@ type RotatingCredentialFieldProps = {
credentialNamesById: Map<string, string>;
title: string;
description: string;
+ showPrimaryBadge?: boolean;
+ fallbackContent?: ReactNode;
};
function formatCredentialSelectionStrategy(
@@ -38,6 +42,8 @@ function RotatingCredentialField({
credentialNamesById,
title,
description,
+ showPrimaryBadge = false,
+ fallbackContent,
}: RotatingCredentialFieldProps) {
const credentialIds = getRotatingCredentialIds(parameter);
const forcedCredentialId = typeof value === "string" ? value : "";
@@ -127,27 +133,38 @@ function RotatingCredentialField({
</Badge>
)}
</div>
- <Select
- disabled={mode !== "force"}
- value={forceSelectValue}
- onValueChange={(value) => onChange(value)}
- >
- <SelectTrigger className="w-full">
- <SelectValue placeholder="Select a credential" />
- </SelectTrigger>
- <SelectContent>
- {credentialIds.map((credentialId) => (
- <SelectItem key={credentialId} value={credentialId}>
- {credentialNamesById.get(credentialId) ??
- credentialId}
- </SelectItem>
- ))}
- </SelectContent>
- </Select>
+ <div className="flex items-center gap-2">
+ <Select
+ disabled={mode !== "force"}
+ value={forceSelectValue}
+ onValueChange={(value) => onChange(value)}
+ >
+ <SelectTrigger className="w-full">
+ <SelectValue placeholder="Select a credential" />
+ </SelectTrigger>
+ <SelectContent>
+ {credentialIds.map((credentialId) => (
+ <SelectItem key={credentialId} value={credentialId}>
+ {credentialNamesById.get(credentialId) ??
+ credentialId}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ {showPrimaryBadge && (
+ <Badge
+ variant="outline"
+ className="shrink-0 text-xs font-normal"
+ >
+ Primary
+ </Badge>
+ )}
+ </div>
</div>
</label>
</RadioGroup>
</FormControl>
+ {fallbackContent}
</div>
</div>
</FormItem>
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.test.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.test.tsx
index 94cd05ebf..054e04eab 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.test.tsx
@@ -244,3 +244,140 @@ describe("LoginBlockCredentialSelector", () => {
);
});
});
+
+describe("LoginBlockCredentialSelector fallback picker", () => {
+ beforeEach(() => {
+ mocks.useCredentialsQuery.mockReturnValue({
+ data: [
+ credential("cred_primary", "primary_credential"),
+ credential("cred_backup_1", "backup_credential_1"),
+ ],
+ isFetching: false,
+ isLoading: false,
+ });
+ useWorkflowParametersStore.setState({
+ parameters: [
+ {
+ key: "portal_credential",
+ parameterType: "credential",
+ credentialId: "cred_primary",
+ },
+ ],
+ });
+ });
+
+ function renderSelector() {
+ return renderInCloud(
+ <LoginBlockCredentialSelector
+ nodeId="node-1"
+ value="portal_credential"
+ />,
+ );
+ }
+
+ async function openFallbackPicker() {
+ renderSelector();
+ fireEvent.click(screen.getByText("Add fallback credentials"));
+ const trigger = await screen.findByRole("button", {
+ name: /add fallback credentials/i,
+ });
+ fireEvent.click(trigger);
+ await screen.findByPlaceholderText("Search credentials...");
+ }
+
+ it("hides fallback controls when credential rotation is configured", () => {
+ useWorkflowParametersStore.setState({
+ parameters: [
+ {
+ key: "portal_credential",
+ parameterType: "credential",
+ credentialId: "cred_primary",
+ credentialIds: ["cred_primary", "cred_backup_1"],
+ },
+ ],
+ });
+
+ renderSelector();
+
+ expect(screen.queryByText("Fallback credentials")).toBeNull();
+ expect(screen.queryByText("Add fallback credentials")).toBeNull();
+ expect(
+ screen.getByText(
+ "Fallback credentials can't be combined with credential rotation.",
+ ),
+ ).toBeTruthy();
+ });
+
+ it("hides credential rotation when fallback credentials are configured", () => {
+ useWorkflowParametersStore.setState({
+ parameters: [
+ {
+ key: "portal_credential",
+ parameterType: "credential",
+ credentialId: "cred_primary",
+ fallbackCredentialIds: ["cred_backup_1"],
+ },
+ ],
+ });
+
+ renderSelector();
+
+ expect(
+ screen.queryByText("Rotate between multiple credentials"),
+ ).toBeNull();
+ expect(
+ screen.getByText(
+ "Credential rotation can't be combined with fallback credentials.",
+ ),
+ ).toBeTruthy();
+ });
+
+ it("Escape closes only the picker and does not reach window listeners", async () => {
+ await openFallbackPicker();
+
+ const windowEscapeSpy = vi.fn();
+ const listener = (event: KeyboardEvent) => {
+ if (event.key === "Escape") {
+ windowEscapeSpy();
+ }
+ };
+ window.addEventListener("keydown", listener);
+ try {
+ fireEvent.keyDown(screen.getByPlaceholderText("Search credentials..."), {
+ key: "Escape",
+ });
+
+ await waitFor(() => {
+ expect(
+ screen.queryByPlaceholderText("Search credentials..."),
+ ).toBeNull();
+ });
+ expect(windowEscapeSpy).not.toHaveBeenCalled();
+ } finally {
+ window.removeEventListener("keydown", listener);
+ }
+ });
+
+ it("selecting a fallback option writes it to the parameter store", async () => {
+ await openFallbackPicker();
+
+ fireEvent.click(await screen.findByText("backup_credential_1"));
+
+ await waitFor(() => {
+ const parameter = useWorkflowParametersStore
+ .getState()
+ .parameters.find((p) => p.key === "portal_credential");
+ expect(parameter).toMatchObject({
+ fallbackCredentialIds: ["cred_backup_1"],
+ });
+ });
+ });
+
+ it("the picker popover is modal so outside clicks cannot hit the canvas", async () => {
+ await openFallbackPicker();
+
+ await waitFor(() => {
+ expect(document.body.style.pointerEvents).toBe("none");
+ });
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx
index 3c41832fd..e2de6b242 100644
--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx
@@ -12,6 +12,13 @@ import {
PopoverContent,
PopoverTrigger,
} from "@/components/ui/popover";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
import { Skeleton } from "@/components/ui/skeleton";
import { Switch } from "@/components/ui/switch";
import { useCredentialsQuery } from "@/routes/workflows/hooks/useCredentialsQuery";
@@ -26,10 +33,12 @@ import {
ChevronDownIcon,
Cross2Icon,
ExclamationTriangleIcon,
+ PlusIcon,
ReloadIcon,
} from "@radix-ui/react-icons";
import type { CredentialApiResponse } from "@/api/types";
import { getHostname } from "@/util/getHostname";
+import { type CredentialFallbackTrigger } from "@/routes/workflows/types/workflowTypes";
import {
CredentialModalTypes,
useCredentialModalState,
@@ -50,6 +59,7 @@ import {
parameterIsAzureVaultCredential,
isAutoGeneratedCredentialKey,
computeWrappedCredentialIds,
+ type SkyvernCredential,
} from "../../types";
import { cn } from "@/util/utils";
import { keepPreviousData } from "@tanstack/react-query";
@@ -65,6 +75,7 @@ import {
} from "@/routes/workflows/components/CredentialCombobox";
const ROTATION_SELECTION_STRATEGY = "round_robin" as const;
+const DEFAULT_FALLBACK_TRIGGER = "credential_failures" as const;
type Props = {
nodeId: string;
@@ -80,7 +91,7 @@ type Props = {
currentInstructions?: string;
};
-type RotationCredentialRowProps = {
+type CredentialListRowProps = {
credentialId: string;
credential: CredentialApiResponse | undefined;
index: number;
@@ -89,14 +100,14 @@ type RotationCredentialRowProps = {
onRemove: () => void;
};
-function RotationCredentialRow({
+function CredentialListRow({
credentialId,
credential,
index,
editable,
canRemove,
onRemove,
-}: RotationCredentialRowProps) {
+}: CredentialListRowProps) {
const detailQuery = useCredentialQuery(credentialId, {
enabled: !credential,
});
@@ -172,6 +183,7 @@ function LoginBlockCredentialSelector({
const [rotationDraft, setRotationDraft] = useState(false);
const [rotationQuery, setRotationQuery] = useState("");
const [debouncedRotationQuery] = useDebounce(rotationQuery, 300);
+ const [fallbackDraft, setFallbackDraft] = useState(false);
const {
parameters: workflowParameters,
setParameters: setWorkflowParameters,
@@ -203,6 +215,7 @@ function LoginBlockCredentialSelector({
useEffect(() => {
setRotationDraft(false);
+ setFallbackDraft(false);
}, [value]);
// Determine which credential is currently selected (by credential_id)
@@ -282,6 +295,14 @@ function LoginBlockCredentialSelector({
});
}, [selectedSkyvernCredentialParameter, selectedCredentialId]);
+ const fallbackCredentialIds = useMemo(() => {
+ return (
+ selectedSkyvernCredentialParameter?.fallbackCredentialIds ?? []
+ ).filter((id, index, ids): id is string => {
+ return typeof id === "string" && ids.indexOf(id) === index;
+ });
+ }, [selectedSkyvernCredentialParameter]);
+
const rotationIsSaved = rotationCredentialIds.length > 1;
const rotationMode =
Boolean(selectedSkyvernCredentialParameter) &&
@@ -295,6 +316,10 @@ function LoginBlockCredentialSelector({
search: debouncedRotationQuery.trim() || undefined,
placeholderData: keepPreviousData,
});
+ const fallbackIsSaved = fallbackCredentialIds.length > 0;
+ const fallbackMode =
+ Boolean(selectedSkyvernCredentialParameter) &&
+ (fallbackDraft || fallbackIsSaved);
const workflowStartNode = useMemo<
{ id: string; data: WorkflowStartNodeData } | undefined
@@ -344,6 +369,19 @@ function LoginBlockCredentialSelector({
browserProfileUrl: credential.tested_url ?? null,
}));
+ const fallbackCredentialOptions = credentials
+ .filter(
+ (credential) =>
+ !fallbackCredentialIds.includes(credential.credential_id) &&
+ credential.credential_id !== selectedCredentialId,
+ )
+ .map((credential) => ({
+ label: credential.name,
+ value: credential.credential_id,
+ hasBrowserProfile: !!credential.browser_profile_id,
+ browserProfileUrl: credential.tested_url ?? null,
+ }));
+
// Auto-generated Skyvern wrappers are hidden (they back a direct credential
// pick); user-authored Skyvern params and external vault params are listed.
// Workflow credential_id params are authored in the Parameters panel, not here.
@@ -419,6 +457,70 @@ function LoginBlockCredentialSelector({
setHasChanges(true);
};
+ const writeFallbackCredentialIds = (credentialIds: Array<string>) => {
+ if (!selectedSkyvernCredentialParameter) {
+ return;
+ }
+
+ const orderedCredentialIds = credentialIds.filter((id, index, ids) => {
+ return ids.indexOf(id) === index;
+ });
+
+ const newParameters = workflowParameters.map((parameter) => {
+ if (
+ parameter.key !== selectedSkyvernCredentialParameter.key ||
+ parameter.parameterType !== "credential" ||
+ !parameterIsSkyvernCredential(parameter)
+ ) {
+ return parameter;
+ }
+
+ if (orderedCredentialIds.length === 0) {
+ return {
+ ...parameter,
+ fallbackCredentialIds: null,
+ fallbackTrigger: null,
+ };
+ }
+
+ return {
+ ...parameter,
+ fallbackCredentialIds: orderedCredentialIds,
+ fallbackTrigger: parameter.fallbackTrigger ?? null,
+ };
+ });
+
+ setWorkflowParameters(newParameters);
+ setHasChanges(true);
+ };
+
+ const writeFallbackTrigger = (fallbackTrigger: CredentialFallbackTrigger) => {
+ if (
+ !selectedSkyvernCredentialParameter ||
+ fallbackCredentialIds.length === 0
+ ) {
+ return;
+ }
+
+ const newParameters = workflowParameters.map((parameter) => {
+ if (
+ parameter.key !== selectedSkyvernCredentialParameter.key ||
+ parameter.parameterType !== "credential" ||
+ !parameterIsSkyvernCredential(parameter)
+ ) {
+ return parameter;
+ }
+
+ return {
+ ...parameter,
+ fallbackTrigger,
+ };
+ });
+
+ setWorkflowParameters(newParameters);
+ setHasChanges(true);
+ };
+
const toggleRotationCredential = (credentialId: string) => {
if (!editable) return;
const nextCredentialIds = rotationCredentialIds.includes(credentialId)
@@ -435,6 +537,22 @@ function LoginBlockCredentialSelector({
}
};
+ const addFallbackCredential = (credentialId: string) => {
+ if (!editable || fallbackCredentialIds.includes(credentialId)) return;
+ writeFallbackCredentialIds([...fallbackCredentialIds, credentialId]);
+ };
+
+ const removeFallbackCredential = (credentialId: string) => {
+ if (!editable) return;
+ const nextCredentialIds = fallbackCredentialIds.filter(
+ (id) => id !== credentialId,
+ );
+ writeFallbackCredentialIds(nextCredentialIds);
+ if (nextCredentialIds.length === 0) {
+ setFallbackDraft(false);
+ }
+ };
+
const disableRotation = () => {
if (!editable) return;
if (rotationCredentialIds.length > 0) {
@@ -464,17 +582,30 @@ function LoginBlockCredentialSelector({
const loginNodes = nodes
.filter((node) => node.id !== nodeId)
.filter(isLoginNode);
- const currentParameter = workflowParameters.find((parameter) => {
- if (parameter.parameterType !== "credential") return false;
- if (!parameterIsSkyvernCredential(parameter)) return false;
- if (!isAutoGeneratedCredentialKey(parameter.key)) return false;
- return parameter.key === value;
- });
+ const currentParameter = workflowParameters.find(
+ (parameter): parameter is SkyvernCredential => {
+ if (parameter.parameterType !== "credential") return false;
+ if (!parameterIsSkyvernCredential(parameter)) return false;
+ if (!isAutoGeneratedCredentialKey(parameter.key)) return false;
+ return parameter.key === value;
+ },
+ );
const isUsedInOtherLoginNodes =
value &&
loginNodes.some((node) => node.data.parameterKeys.includes(value));
+ const deleteOldParameter = currentParameter && !isUsedInOtherLoginNodes;
- if (currentParameter && !isUsedInOtherLoginNodes) {
+ // The new primary may not be its own fallback.
+ const carriedFallbackCredentialIds = deleteOldParameter
+ ? (currentParameter.fallbackCredentialIds ?? []).filter(
+ (credentialId) => credentialId !== newValue,
+ )
+ : [];
+ const carriedFallbackTrigger = deleteOldParameter
+ ? (currentParameter.fallbackTrigger ?? null)
+ : null;
+
+ if (deleteOldParameter) {
newParameters = newParameters.filter(
(parameter) => parameter.key !== value,
);
@@ -495,7 +626,30 @@ function LoginBlockCredentialSelector({
});
if (existingParameter) {
+ // Reuse the existing parameter; carry the outgoing wrapper's fallback
+ // config onto it when it has none of its own.
parameterKeyToUse = existingParameter.key;
+ if (
+ carriedFallbackCredentialIds.length > 0 &&
+ existingParameter.parameterType === "credential" &&
+ parameterIsSkyvernCredential(existingParameter) &&
+ (existingParameter.fallbackCredentialIds?.length ?? 0) === 0
+ ) {
+ newParameters = newParameters.map((parameter) => {
+ if (
+ parameter.key !== existingParameter.key ||
+ parameter.parameterType !== "credential" ||
+ !parameterIsSkyvernCredential(parameter)
+ ) {
+ return parameter;
+ }
+ return {
+ ...parameter,
+ fallbackCredentialIds: carriedFallbackCredentialIds,
+ fallbackTrigger: carriedFallbackTrigger,
+ };
+ });
+ }
} else {
const existingKeys = newParameters.map((parameter) => parameter.key);
const newKey = generateDefaultCredentialParameterKey(existingKeys);
@@ -506,6 +660,10 @@ function LoginBlockCredentialSelector({
parameterType: "credential",
credentialId: newValue,
key: newKey,
+ ...(carriedFallbackCredentialIds.length > 0 && {
+ fallbackCredentialIds: carriedFallbackCredentialIds,
+ fallbackTrigger: carriedFallbackTrigger,
+ }),
},
];
}
@@ -561,11 +719,155 @@ function LoginBlockCredentialSelector({
return undefined;
};
+ const fallbackTrigger =
+ selectedSkyvernCredentialParameter?.fallbackTrigger ??
+ DEFAULT_FALLBACK_TRIGGER;
+
+ const fallbackSection = selectedSkyvernCredentialParameter ? (
+ fallbackMode ? (
+ <div className="mt-3 space-y-2 border-t border-border pt-3">
+ <div className="space-y-1">
+ <span className="text-xs text-foreground dark:text-slate-300">
+ Fallback credentials
+ </span>
+ <p className="text-[11px] leading-4 text-muted-foreground dark:text-slate-500">
+ If a run fails, Skyvern automatically retries it with the next
+ fallback credential.
+ </p>
+ </div>
+ {/* modal: an outside click must only dismiss the picker — non-modal
+ popovers let the same click hit the canvas pane, which deselects
+ the block and closes the whole config panel. */}
+ <Popover modal>
+ <PopoverTrigger asChild>
+ <Button
+ type="button"
+ variant="outline"
+ className="h-auto min-h-10 w-full justify-between px-3 py-2 text-left text-xs font-normal"
+ disabled={!editable || fallbackCredentialOptions.length === 0}
+ >
+ <span className="truncate">
+ {fallbackCredentialIds.length > 0
+ ? `${fallbackCredentialIds.length} fallback ${
+ fallbackCredentialIds.length === 1
+ ? "credential"
+ : "credentials"
+ }`
+ : "Add fallback credentials"}
+ </span>
+ <ChevronDownIcon className="ml-2 size-4 shrink-0 text-muted-foreground" />
+ </Button>
+ </PopoverTrigger>
+ <PopoverContent
+ className="w-[28rem] p-0"
+ align="start"
+ // Escape must only dismiss the picker; without this it bubbles to
+ // the editor's window listener, which closes the whole block panel.
+ onEscapeKeyDown={(event) => event.stopPropagation()}
+ >
+ <Command>
+ <CommandInput placeholder="Search credentials..." />
+ <CommandList>
+ <CommandEmpty>No credentials found.</CommandEmpty>
+ <CommandGroup>
+ {fallbackCredentialOptions.map((option) => (
+ <CommandItem
+ key={option.value}
+ className="cursor-pointer"
+ onSelect={() => addFallbackCredential(option.value)}
+ >
+ <div
+ className={cn(
+ "mr-2 flex size-4 items-center justify-center rounded-sm border border-primary",
+ "opacity-50 [&_svg]:invisible",
+ )}
+ >
+ <CheckIcon className="size-4" />
+ </div>
+ <div className="flex min-w-0 items-center gap-2">
+ <span className="truncate">{option.label}</span>
+ {option.hasBrowserProfile ? (
+ <span className="rounded bg-green-100 px-1.5 py-0.5 text-[10px] text-green-700 dark:bg-green-900/40 dark:text-green-400">
+ saved-profile
+ </span>
+ ) : null}
+ {option.browserProfileUrl ? (
+ <span className="truncate text-[10px] text-muted-foreground">
+ {getHostname(option.browserProfileUrl)}
+ </span>
+ ) : null}
+ </div>
+ </CommandItem>
+ ))}
+ </CommandGroup>
+ </CommandList>
+ </Command>
+ </PopoverContent>
+ </Popover>
+
+ {fallbackCredentialIds.length > 0 ? (
+ <>
+ <div className="space-y-1">
+ {fallbackCredentialIds.map((credentialId, index) => (
+ <CredentialListRow
+ key={credentialId}
+ credentialId={credentialId}
+ credential={credentials.find(
+ (credential) => credential.credential_id === credentialId,
+ )}
+ index={index}
+ editable={editable}
+ canRemove
+ onRemove={() => removeFallbackCredential(credentialId)}
+ />
+ ))}
+ </div>
+ <div className="space-y-1">
+ <span className="text-xs text-foreground dark:text-slate-300">
+ Retry failed runs
+ </span>
+ <Select
+ disabled={!editable}
+ value={fallbackTrigger}
+ onValueChange={(value) =>
+ writeFallbackTrigger(value as CredentialFallbackTrigger)
+ }
+ >
+ <SelectTrigger className="h-9 w-full text-xs">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="credential_failures">
+ On credential failures
+ </SelectItem>
+ <SelectItem value="any_failure">On any failure</SelectItem>
+ </SelectContent>
+ </Select>
+ </div>
+ </>
+ ) : null}
+ </div>
+ ) : (
+ <button
+ type="button"
+ className="mt-2 inline-flex items-center gap-1 text-xs text-muted-foreground transition-colors hover:text-foreground disabled:pointer-events-none disabled:opacity-50 dark:hover:text-slate-200"
+ disabled={!editable}
+ onClick={() => setFallbackDraft(true)}
+ >
+ <PlusIcon className="size-3" />
+ Add fallback credentials
+ </button>
+ )
+ ) : null;
+
if (rotationMode) {
return (
<>
<div className="space-y-2">
- <Popover>
+ {/* modal: an outside click must only dismiss the picker — non-modal
+ popovers let the same click hit the canvas pane, which deselects
+ the block and closes the whole config panel. */}
+ <Popover modal>
<PopoverTrigger asChild>
<Button
type="button"
@@ -581,7 +883,13 @@ function LoginBlockCredentialSelector({
<ChevronDownIcon className="ml-2 size-4 shrink-0 text-muted-foreground" />
</Button>
</PopoverTrigger>
- <PopoverContent className="w-[28rem] p-0" align="start">
+ <PopoverContent
+ className="w-[28rem] p-0"
+ align="start"
+ // Escape must only dismiss the picker; without this it bubbles to
+ // the editor's window listener, which closes the whole block panel.
+ onEscapeKeyDown={(event) => event.stopPropagation()}
+ >
<Command shouldFilter={false}>
<CommandInput
placeholder="Search credentials..."
@@ -643,7 +951,7 @@ function LoginBlockCredentialSelector({
<div className="space-y-1">
{rotationCredentialIds.map((credentialId, index) => (
- <RotationCredentialRow
+ <CredentialListRow
key={credentialId}
credentialId={credentialId}
credential={rotationCredentials.find(
@@ -687,6 +995,10 @@ function LoginBlockCredentialSelector({
>
Use single credential
</button>
+ <p className="text-[11px] leading-4 text-muted-foreground dark:text-slate-500">
+ Fallback credentials can&apos;t be combined with credential
+ rotation.
+ </p>
</div>
<CredentialsModal
onCredentialCreated={(id) => {
@@ -734,10 +1046,13 @@ function LoginBlockCredentialSelector({
: "h-auto min-h-10 w-full",
}}
/>
- {!rotationMode &&
- selectedSkyvernCredentialParameter &&
- selectedCredentialId &&
- !isCredentialMissing ? (
+ {fallbackMode ? (
+ <p className="mt-2 text-[11px] leading-4 text-muted-foreground dark:text-slate-500">
+ Credential rotation can&apos;t be combined with fallback credentials.
+ </p>
+ ) : selectedSkyvernCredentialParameter &&
+ selectedCredentialId &&
+ !isCredentialMissing ? (
<button
type="button"
className="mt-2 inline-flex items-center gap-1 text-xs text-muted-foreground transition-colors hover:text-foreground disabled:pointer-events-none disabled:opacity-50 dark:hover:text-slate-200"
@@ -748,6 +1063,7 @@ function LoginBlockCredentialSelector({
Rotate between multiple credentials
</button>
) : null}
+ {fallbackSection}
<CredentialsModal
onCredentialCreated={(id) => {
if (!editable) return;
diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx
index 64856b608..27ddf8f36 100644
--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx
@@ -33,6 +33,7 @@ import {
parameterIsAzureVaultCredential,
AUTO_GENERATED_CREDENTIAL_KEY_PATTERN,
} from "../types";
+import { applySkyvernCredentialEdit } from "../utils";
import { getDefaultValueForParameterType } from "../workflowEditorUtils";
import { validateBitwardenLoginCredential } from "./util";
import { HelpTooltip } from "@/components/HelpTooltip";
@@ -1173,12 +1174,13 @@ function WorkflowParameterEditPanel({
});
return;
}
- onSave({
- key,
- parameterType: "credential",
- credentialId,
- description,
- });
+ onSave(
+ applySkyvernCredentialEdit(initialValues, {
+ key,
+ credentialId,
+ description,
+ }),
+ );
return;
}
diff --git a/skyvern-frontend/src/routes/workflows/editor/types.ts b/skyvern-frontend/src/routes/workflows/editor/types.ts
index 7b9864aa5..7053beea2 100644
--- a/skyvern-frontend/src/routes/workflows/editor/types.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/types.ts
@@ -1,4 +1,5 @@
import {
+ CredentialFallbackTrigger,
CredentialSelectionStrategy,
WorkflowParameterValueType,
} from "../types/workflowTypes";
@@ -25,6 +26,8 @@ export type SkyvernCredential = {
credentialId: string;
credentialIds?: Array<string> | null;
selectionStrategy?: CredentialSelectionStrategy | null;
+ fallbackCredentialIds?: Array<string> | null;
+ fallbackTrigger?: CredentialFallbackTrigger | null;
dataType?: typeof WorkflowParameterValueType.CredentialId;
};
diff --git a/skyvern-frontend/src/routes/workflows/editor/utils.test.ts b/skyvern-frontend/src/routes/workflows/editor/utils.test.ts
index 7efa34fe4..648b3da70 100644
--- a/skyvern-frontend/src/routes/workflows/editor/utils.test.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/utils.test.ts
@@ -5,6 +5,7 @@ import {
WorkflowParameterValueType,
} from "../types/workflowTypes";
import {
+ applySkyvernCredentialEdit,
getInitialParameters,
skyvernCredentialToParameterYAML,
} from "./utils";
@@ -81,6 +82,8 @@ describe("getInitialParameters", () => {
key: "static_credential",
description: null,
credential_id: "cred_static",
+ fallback_credential_ids: ["cred_backup"],
+ fallback_trigger: "any_failure",
created_at: "2026-06-23T00:00:00Z",
modified_at: "2026-06-23T00:00:00Z",
deleted_at: null,
@@ -89,17 +92,21 @@ describe("getInitialParameters", () => {
},
};
- expect(getInitialParameters(workflow)).toEqual([
+ const initialParameters = getInitialParameters(workflow);
+
+ expect(initialParameters).toEqual([
expect.objectContaining({
key: "credential_id",
parameterType: "credential",
dataType: WorkflowParameterValueType.CredentialId,
}),
- expect.not.objectContaining({
+ expect.objectContaining({
key: "static_credential",
- dataType: WorkflowParameterValueType.CredentialId,
+ fallbackCredentialIds: ["cred_backup"],
+ fallbackTrigger: "any_failure",
}),
]);
+ expect(initialParameters[1]).not.toHaveProperty("dataType");
});
});
@@ -151,8 +158,148 @@ describe("skyvernCredentialToParameterYAML", () => {
credential_id: "cred_1",
credential_ids: ["cred_1", "cred_2"],
selection_strategy: "round_robin",
+ fallback_credential_ids: null,
+ fallback_trigger: null,
+ key: "credentials",
+ description: null,
+ });
+ });
+
+ test("serializes a single credential with fallbacks as a credential parameter", () => {
+ const parameter: SkyvernCredential = {
+ key: "credentials",
+ parameterType: "credential",
+ credentialId: "cred_1",
+ fallbackCredentialIds: ["cred_backup"],
+ fallbackTrigger: "credential_failures",
+ };
+
+ expect(skyvernCredentialToParameterYAML(parameter)).toEqual({
+ parameter_type: "credential",
+ credential_id: "cred_1",
+ credential_ids: null,
+ selection_strategy: null,
+ fallback_credential_ids: ["cred_backup"],
+ fallback_trigger: "credential_failures",
key: "credentials",
description: null,
});
});
});
+
+describe("applySkyvernCredentialEdit", () => {
+ const previous: SkyvernCredential = {
+ key: "portal_credential",
+ parameterType: "credential",
+ credentialId: "cred_primary",
+ fallbackCredentialIds: ["cred_backup_1", "cred_backup_2"],
+ fallbackTrigger: "any_failure",
+ dataType: WorkflowParameterValueType.CredentialId,
+ };
+
+ test("preserves fallback config on an edit that only touches visible fields", () => {
+ expect(
+ applySkyvernCredentialEdit(previous, {
+ key: "portal_credential",
+ credentialId: "cred_primary",
+ description: "updated description",
+ }),
+ ).toEqual({
+ key: "portal_credential",
+ parameterType: "credential",
+ credentialId: "cred_primary",
+ description: "updated description",
+ credentialIds: null,
+ selectionStrategy: null,
+ fallbackCredentialIds: ["cred_backup_1", "cred_backup_2"],
+ fallbackTrigger: "any_failure",
+ dataType: WorkflowParameterValueType.CredentialId,
+ });
+ });
+
+ test("removes the new primary from the fallback list on a primary change", () => {
+ const edited = applySkyvernCredentialEdit(previous, {
+ key: "portal_credential",
+ credentialId: "cred_backup_1",
+ });
+
+ expect(edited.credentialId).toBe("cred_backup_1");
+ expect(edited.fallbackCredentialIds).toEqual(["cred_backup_2"]);
+ expect(edited.fallbackTrigger).toBe("any_failure");
+ });
+
+ test("clears the trigger when the primary change empties the fallback list", () => {
+ const edited = applySkyvernCredentialEdit(
+ { ...previous, fallbackCredentialIds: ["cred_backup_1"] },
+ { key: "portal_credential", credentialId: "cred_backup_1" },
+ );
+
+ expect(edited.fallbackCredentialIds).toBeNull();
+ expect(edited.fallbackTrigger).toBeNull();
+ });
+
+ test("preserves a rotation pool, swapping the new primary into its head", () => {
+ const edited = applySkyvernCredentialEdit(
+ {
+ ...previous,
+ credentialIds: ["cred_primary", "cred_rotate_1", "cred_rotate_2"],
+ selectionStrategy: "round_robin",
+ },
+ { key: "portal_credential", credentialId: "cred_new" },
+ );
+
+ expect(edited.credentialIds).toEqual([
+ "cred_new",
+ "cred_rotate_1",
+ "cred_rotate_2",
+ ]);
+ expect(edited.selectionStrategy).toBe("round_robin");
+ });
+
+ test("collapses a rotation pool that shrinks below two credentials", () => {
+ const edited = applySkyvernCredentialEdit(
+ {
+ ...previous,
+ credentialIds: ["cred_primary", "cred_rotate_1"],
+ selectionStrategy: "round_robin",
+ },
+ { key: "portal_credential", credentialId: "cred_rotate_1" },
+ );
+
+ expect(edited.credentialIds).toBeNull();
+ expect(edited.selectionStrategy).toBeNull();
+ });
+
+ test("returns a plain credential parameter when adding (no previous value)", () => {
+ expect(
+ applySkyvernCredentialEdit(undefined, {
+ key: "credentials",
+ credentialId: "cred_1",
+ }),
+ ).toEqual({
+ key: "credentials",
+ parameterType: "credential",
+ credentialId: "cred_1",
+ description: null,
+ });
+ });
+
+ test("returns a plain credential parameter when the previous value is not a Skyvern credential", () => {
+ expect(
+ applySkyvernCredentialEdit(
+ {
+ key: "some_input",
+ parameterType: "workflow",
+ dataType: WorkflowParameterValueType.String,
+ defaultValue: "x",
+ },
+ { key: "some_input", credentialId: "cred_1" },
+ ),
+ ).toEqual({
+ key: "some_input",
+ parameterType: "credential",
+ credentialId: "cred_1",
+ description: null,
+ });
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/editor/utils.ts b/skyvern-frontend/src/routes/workflows/editor/utils.ts
index 121de56b1..7e164559d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/utils.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/utils.ts
@@ -10,7 +10,11 @@ import {
CredentialParameterYAML,
WorkflowParameterYAML,
} from "../types/workflowYamlTypes";
-import { ParametersState, SkyvernCredential } from "./types";
+import {
+ ParametersState,
+ parameterIsSkyvernCredential,
+ SkyvernCredential,
+} from "./types";
/**
* A single credential serializes to an editable workflow `credential_id`
@@ -21,7 +25,9 @@ function skyvernCredentialToParameterYAML(
parameter: SkyvernCredential,
): WorkflowParameterYAML | CredentialParameterYAML {
const hasCredentialRotation = (parameter.credentialIds?.length ?? 0) >= 2;
- if (!hasCredentialRotation) {
+ const hasFallbackCredentials =
+ (parameter.fallbackCredentialIds?.length ?? 0) > 0;
+ if (!hasCredentialRotation && !hasFallbackCredentials) {
return {
parameter_type: WorkflowParameterTypes.Workflow,
workflow_parameter_type: WorkflowParameterValueType.CredentialId,
@@ -35,11 +41,67 @@ function skyvernCredentialToParameterYAML(
credential_id: parameter.credentialId,
credential_ids: parameter.credentialIds ?? null,
selection_strategy: parameter.selectionStrategy ?? null,
+ fallback_credential_ids: parameter.fallbackCredentialIds ?? null,
+ fallback_trigger: parameter.fallbackTrigger ?? null,
key: parameter.key,
description: parameter.description || null,
};
}
+/**
+ * The parameter edit panel only exposes key/credential/description, so an edit
+ * must carry over the rotation pool and fallback config it can't see — a plain
+ * rebuild silently wipes them. When the primary credential changes it is
+ * removed from the fallback list (a credential may not be its own fallback)
+ * and swapped into the head of the rotation pool.
+ */
+function applySkyvernCredentialEdit(
+ previous: ParametersState[number] | null | undefined,
+ edit: { key: string; credentialId: string; description?: string | null },
+): SkyvernCredential {
+ const base: SkyvernCredential = {
+ key: edit.key,
+ parameterType: "credential",
+ credentialId: edit.credentialId,
+ description: edit.description ?? null,
+ };
+ if (
+ !previous ||
+ previous.parameterType !== "credential" ||
+ !parameterIsSkyvernCredential(previous)
+ ) {
+ return base;
+ }
+
+ const previousPool = previous.credentialIds ?? [];
+ const rotationPool =
+ previousPool.length >= 2
+ ? [
+ edit.credentialId,
+ ...previousPool.filter(
+ (id) => id !== previous.credentialId && id !== edit.credentialId,
+ ),
+ ]
+ : [];
+ const credentialIds = rotationPool.length >= 2 ? rotationPool : null;
+
+ const fallbackCredentialIds = (previous.fallbackCredentialIds ?? []).filter(
+ (id) => id !== edit.credentialId,
+ );
+ const hasFallback = fallbackCredentialIds.length > 0;
+
+ return {
+ ...base,
+ credentialIds,
+ selectionStrategy: credentialIds
+ ? (previous.selectionStrategy ?? null)
+ : null,
+ fallbackCredentialIds: hasFallback ? fallbackCredentialIds : null,
+ fallbackTrigger: hasFallback ? (previous.fallbackTrigger ?? null) : null,
+ dataType: previous.dataType,
+ };
+}
+
const getInitialParameters = (workflow: WorkflowApiResponse) => {
return workflow.workflow_definition.parameters
.filter((parameter) => isDisplayedInWorkflowEditor(parameter))
@@ -103,6 +165,8 @@ const getInitialParameters = (workflow: WorkflowApiResponse) => {
credentialId: parameter.credential_id,
credentialIds: parameter.credential_ids ?? null,
selectionStrategy: parameter.selection_strategy ?? null,
+ fallbackCredentialIds: parameter.fallback_credential_ids ?? null,
+ fallbackTrigger: parameter.fallback_trigger ?? null,
description: parameter.description,
};
} else if (
@@ -202,6 +266,7 @@ const constructCacheKeyValueFromParameters = (opts: {
};
export {
+ applySkyvernCredentialEdit,
constructCacheKeyValue,
constructCacheKeyValueFromParameters,
getInitialParameters,
diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
index 18d04da8f..d6f67a9b1 100644
--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts
@@ -4253,6 +4253,8 @@ function convertParametersToParameterYAML(
credential_id: parameter.credential_id,
credential_ids: parameter.credential_ids ?? null,
selection_strategy: parameter.selection_strategy ?? null,
+ fallback_credential_ids: parameter.fallback_credential_ids ?? null,
+ fallback_trigger: parameter.fallback_trigger ?? null,
};
}
case WorkflowParameterTypes.OnePassword: {
diff --git a/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.test.ts b/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.test.ts
index bc44848aa..b16c05ed7 100644
--- a/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.test.ts
+++ b/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.test.ts
@@ -8,7 +8,11 @@ import {
WorkflowParameterTypes,
WorkflowParameterValueType,
} from "./types/workflowTypes";
-import { getLoginCredentialInputs } from "./runWorkflowCredentials";
+import {
+ getFallbackCredentialIds,
+ getLoginCredentialInputs,
+ getRotatingCredentialIds,
+} from "./runWorkflowCredentials";
function workflowParameter(key: string): WorkflowParameter {
return {
@@ -80,6 +84,8 @@ describe("getLoginCredentialInputs", () => {
{
parameter: loginCredential,
loginBlockLabels: ["Portal login"],
+ fallbackCredentialIds: [],
+ fallbackTrigger: null,
},
]);
});
@@ -103,6 +109,8 @@ describe("getLoginCredentialInputs", () => {
{
parameter: credential,
loginBlockLabels: ["Primary login", "Nested login"],
+ fallbackCredentialIds: [],
+ fallbackTrigger: null,
},
]);
});
@@ -122,8 +130,90 @@ describe("getLoginCredentialInputs", () => {
{
parameter: credential,
loginBlockLabels: ["Rotating login"],
+ fallbackCredentialIds: [],
+ fallbackTrigger: null,
+ },
+ ]);
+ });
+
+ test("returns single-credential parameters that have fallback credentials", () => {
+ const credential: CredentialParameter = {
+ ...rotatingCredentialParameter("account_credential", ["cred_1"]),
+ credential_ids: null,
+ selection_strategy: null,
+ fallback_credential_ids: ["cred_fb1"],
+ };
+
+ const inputs = getLoginCredentialInputs({
+ workflow: workflowWithBlocks(
+ [loginBlock("Fallback login", [credential])],
+ [credential],
+ ),
+ workflowParameters: [],
+ });
+
+ expect(inputs).toEqual([
+ {
+ parameter: credential,
+ loginBlockLabels: ["Fallback login"],
+ fallbackCredentialIds: ["cred_fb1"],
+ fallbackTrigger: "credential_failures",
},
]);
+ expect((inputs[0]!.parameter as CredentialParameter).credential_id).toBe(
+ "cred_1",
+ );
+ });
+
+ test("passes ordered fallback credential ids through and defaults the trigger", () => {
+ const credential: CredentialParameter = {
+ ...rotatingCredentialParameter("account_credential"),
+ fallback_credential_ids: ["cred_fb2", "cred_fb1", "cred_fb2"],
+ };
+
+ const inputs = getLoginCredentialInputs({
+ workflow: workflowWithBlocks(
+ [loginBlock("Rotating login", [credential])],
+ [credential],
+ ),
+ workflowParameters: [],
+ });
+
+ expect(inputs[0]!.fallbackCredentialIds).toEqual(["cred_fb2", "cred_fb1"]);
+ expect(inputs[0]!.fallbackTrigger).toBe("credential_failures");
+ });
+
+ test("preserves an explicit any_failure fallback trigger", () => {
+ const credential: CredentialParameter = {
+ ...rotatingCredentialParameter("account_credential"),
+ fallback_credential_ids: ["cred_fb1"],
+ fallback_trigger: "any_failure",
+ };
+
+ const inputs = getLoginCredentialInputs({
+ workflow: workflowWithBlocks(
+ [loginBlock("Rotating login", [credential])],
+ [credential],
+ ),
+ workflowParameters: [],
+ });
+
+ expect(inputs[0]!.fallbackTrigger).toBe("any_failure");
+ });
+
+ test("classifies fallback-only parameters as single-credential, not rotating", () => {
+ const credential: CredentialParameter = {
+ ...rotatingCredentialParameter("account_credential", ["cred_1"]),
+ credential_ids: null,
+ selection_strategy: null,
+ fallback_credential_ids: ["cred_fb1", "cred_fb2"],
+ };
+
+ expect(getRotatingCredentialIds(credential)).toEqual([]);
+ expect(getFallbackCredentialIds(credential)).toEqual([
+ "cred_fb1",
+ "cred_fb2",
+ ]);
});
test("ignores block-scoped single credentials", () => {
@@ -149,3 +239,40 @@ describe("getLoginCredentialInputs", () => {
expect(inputs).toEqual([]);
});
});
+
+describe("getFallbackCredentialIds", () => {
+ test("deduplicates fallback credential ids while preserving order", () => {
+ const credential = {
+ ...rotatingCredentialParameter("account_credential"),
+ fallback_credential_ids: ["fallback_1", "fallback_2", "fallback_1"],
+ };
+
+ expect(getFallbackCredentialIds(credential)).toEqual([
+ "fallback_1",
+ "fallback_2",
+ ]);
+ });
+
+ test("ignores empty and invalid fallback credential ids", () => {
+ const credential = {
+ ...rotatingCredentialParameter("account_credential"),
+ fallback_credential_ids: [
+ "fallback_1",
+ "",
+ null,
+ "fallback_2",
+ ] as unknown as Array<string>,
+ };
+
+ expect(getFallbackCredentialIds(credential)).toEqual([
+ "fallback_1",
+ "fallback_2",
+ ]);
+ });
+
+ test("returns an empty list when fallback credentials are unset", () => {
+ const credential = rotatingCredentialParameter("account_credential");
+
+ expect(getFallbackCredentialIds(credential)).toEqual([]);
+ });
+});
diff --git a/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.ts b/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.ts
index 62d8eba99..fa7a9e41d 100644
--- a/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.ts
+++ b/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.ts
@@ -1,4 +1,5 @@
import {
+ CredentialFallbackTrigger,
CredentialParameter,
WorkflowApiResponse,
WorkflowParameter,
@@ -10,6 +11,8 @@ import { visitWorkflowBlocks } from "./workflowBlockUtils";
export type LoginCredentialInput = {
parameter: WorkflowParameter | CredentialParameter;
loginBlockLabels: Array<string>;
+ fallbackCredentialIds: Array<string>;
+ fallbackTrigger: CredentialFallbackTrigger | null;
};
function isCredentialIdWorkflowParameter(
@@ -39,7 +42,18 @@ export function getRotatingCredentialIds(
);
}
-function isRotatingCredentialParameter(
+export function getFallbackCredentialIds(
+ parameter: CredentialParameter,
+): Array<string> {
+ return (parameter.fallback_credential_ids ?? []).filter(
+ (credentialId, index, allCredentialIds): credentialId is string =>
+ typeof credentialId === "string" &&
+ credentialId.length > 0 &&
+ allCredentialIds.indexOf(credentialId) === index,
+ );
+}
+
+function isConfigurableCredentialParameter(
parameter: unknown,
): parameter is CredentialParameter {
if (parameter === null || typeof parameter !== "object") {
@@ -47,10 +61,17 @@ function isRotatingCredentialParameter(
}
const maybeParameter = parameter as Partial<CredentialParameter>;
+ if (
+ maybeParameter.parameter_type !== WorkflowParameterTypes.Credential ||
+ typeof maybeParameter.key !== "string"
+ ) {
+ return false;
+ }
+
+ const credentialParameter = maybeParameter as CredentialParameter;
return (
- maybeParameter.parameter_type === WorkflowParameterTypes.Credential &&
- typeof maybeParameter.key === "string" &&
- getRotatingCredentialIds(maybeParameter as CredentialParameter).length > 1
+ getRotatingCredentialIds(credentialParameter).length > 1 ||
+ getFallbackCredentialIds(credentialParameter).length > 0
);
}
@@ -68,7 +89,7 @@ function collectLoginCredentialLabels(
for (const parameter of block.parameters ?? []) {
if (
!isCredentialIdWorkflowParameter(parameter) &&
- !isRotatingCredentialParameter(parameter)
+ !isConfigurableCredentialParameter(parameter)
) {
continue;
}
@@ -102,7 +123,7 @@ export function getLoginCredentialInputs({
[
...workflowParameters.filter(isCredentialIdWorkflowParameter),
...workflow.workflow_definition.parameters.filter(
- isRotatingCredentialParameter,
+ isConfigurableCredentialParameter,
),
].map((parameter) => [parameter.key, parameter]),
);
@@ -123,10 +144,22 @@ export function getLoginCredentialInputs({
return [];
}
+ const fallbackCredentialIds =
+ parameter.parameter_type === WorkflowParameterTypes.Credential
+ ? getFallbackCredentialIds(parameter)
+ : [];
+ const fallbackTrigger =
+ fallbackCredentialIds.length > 0 &&
+ parameter.parameter_type === WorkflowParameterTypes.Credential
+ ? (parameter.fallback_trigger ?? "credential_failures")
+ : null;
+
return [
{
parameter,
loginBlockLabels: Array.from(labels),
+ fallbackCredentialIds,
+ fallbackTrigger,
},
];
});
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
index 24aebfaf5..b8cfbcb54 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts
@@ -86,6 +86,7 @@ export type AzureVaultCredentialParameter = WorkflowParameterBase & {
};
export type CredentialSelectionStrategy = "round_robin" | "random";
+export type CredentialFallbackTrigger = "credential_failures" | "any_failure";
export type CredentialParameter = WorkflowParameterBase & {
parameter_type: "credential";
@@ -94,6 +95,8 @@ export type CredentialParameter = WorkflowParameterBase & {
credential_id: string;
credential_ids?: Array<string> | null;
selection_strategy?: CredentialSelectionStrategy | null;
+ fallback_credential_ids?: Array<string> | null;
+ fallback_trigger?: CredentialFallbackTrigger | null;
created_at: string;
modified_at: string;
deleted_at: string | null;
diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
index c07cea96e..e9498c8eb 100644
--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts
@@ -1,6 +1,7 @@
import { ProxyLocation, RunEngine } from "@/api/types";
import {
CodeBlockStep,
+ CredentialFallbackTrigger,
CredentialSelectionStrategy,
WorkflowBlockType,
WorkflowModel,
@@ -133,6 +134,8 @@ export type CredentialParameterYAML = ParameterYAMLBase & {
credential_id: string;
credential_ids?: Array<string> | null;
selection_strategy?: CredentialSelectionStrategy | null;
+ fallback_credential_ids?: Array<string> | null;
+ fallback_trigger?: CredentialFallbackTrigger | null;
};
export type BlockYAML =
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 9c3d02ea8..17fa33e35 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -719,6 +719,12 @@ class AgentFunction:
def has_proxy_session_extra_http_headers(self, extra_http_headers: dict[str, str] | None) -> bool:
return False
+ def strip_proxy_session_extra_http_headers(
+ self,
+ extra_http_headers: dict[str, str] | None,
+ ) -> dict[str, str] | None:
+ return extra_http_headers
+
def merge_proxy_session_extra_http_headers(
self,
extra_http_headers: dict[str, str] | None,
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index e6bb93103..139a4ca45 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -700,6 +700,13 @@ class WorkflowRunModel(Base):
"queued_at",
postgresql_where=text("status IN ('queued', 'running', 'paused') AND browser_session_id IS NULL"),
),
+ Index(
+ "ix_workflow_runs_retried_from_workflow_run_id",
+ "retried_from_workflow_run_id",
+ unique=True,
+ postgresql_where=text("retried_from_workflow_run_id IS NOT NULL"),
+ sqlite_where=text("retried_from_workflow_run_id IS NOT NULL"),
+ ),
)
workflow_run_id = Column(String, primary_key=True, default=generate_workflow_run_id)
@@ -730,6 +737,8 @@ class WorkflowRunModel(Base):
debug_session_id: Column = Column(String, nullable=True)
trigger_type = Column(String, nullable=True)
workflow_schedule_id = Column(String, nullable=True, index=True)
+ retried_from_workflow_run_id = Column(String, nullable=True)
+ fallback_attempt = Column(Integer, nullable=True)
ai_fallback = Column(Boolean, nullable=True)
code_gen = Column(Boolean, nullable=True)
waiting_for_verification_code = Column(Boolean, nullable=False, default=False, server_default=sqlalchemy.false())
@@ -908,6 +917,8 @@ class CredentialParameterModel(Base):
credential_id = Column(String, nullable=False)
credential_ids = Column(JSON, nullable=True)
selection_strategy = Column(String, nullable=True)
+ fallback_credential_ids = Column(JSON, nullable=True)
+ fallback_trigger = Column(String, nullable=True)
created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)
modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False)
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index 2bca94328..c1254e69e 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -292,6 +292,8 @@ class WorkflowParametersRepository(BaseRepository):
credential_id=parameter.credential_id,
credential_ids=parameter.credential_ids,
selection_strategy=parameter.selection_strategy,
+ fallback_credential_ids=parameter.fallback_credential_ids,
+ fallback_trigger=parameter.fallback_trigger,
deleted_at=parameter.deleted_at,
)
elif isinstance(parameter, OnePasswordCredentialParameter):
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index 659894b81..abfc7dab3 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -208,6 +208,8 @@ class WorkflowRunsRepository(BaseRepository):
workflow_run_id: str | None = None,
trigger_type: WorkflowRunTriggerType | None = None,
workflow_schedule_id: str | None = None,
+ retried_from_workflow_run_id: str | None = None,
+ fallback_attempt: int | None = None,
ignore_inherited_workflow_system_prompt: bool = False,
copilot_session_id: str | None = None,
) -> WorkflowRun:
@@ -239,6 +241,8 @@ class WorkflowRunsRepository(BaseRepository):
code_gen=code_gen,
trigger_type=trigger_type.value if trigger_type else None,
workflow_schedule_id=workflow_schedule_id,
+ retried_from_workflow_run_id=retried_from_workflow_run_id,
+ fallback_attempt=fallback_attempt,
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
copilot_session_id=copilot_session_id,
**kwargs,
@@ -248,6 +252,18 @@ class WorkflowRunsRepository(BaseRepository):
await session.refresh(workflow_run)
return convert_to_workflow_run(workflow_run)
+ @db_operation("get_workflow_run_retried_by")
+ async def get_workflow_run_retried_by(self, workflow_run_id: str, organization_id: str) -> str | None:
+ async with self.Session() as session:
+ query = (
+ select(WorkflowRunModel.workflow_run_id)
+ .where(WorkflowRunModel.retried_from_workflow_run_id == workflow_run_id)
+ .where(WorkflowRunModel.organization_id == organization_id)
+ .order_by(WorkflowRunModel.created_at.desc())
+ .limit(1)
+ )
+ return (await session.scalars(query)).first()
+
@db_operation("update_workflow_run")
async def update_workflow_run(
self,
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index 4d5b86083..c91169186 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -613,6 +613,8 @@ def convert_to_workflow_run(
browser_profile_id=workflow_run_model.browser_profile_id,
status=WorkflowRunStatus[workflow_run_model.status],
failure_reason=workflow_run_model.failure_reason,
+ retried_from_workflow_run_id=workflow_run_model.retried_from_workflow_run_id,
+ fallback_attempt=workflow_run_model.fallback_attempt,
proxy_location=deserialize_proxy_location(workflow_run_model.proxy_location),
webhook_callback_url=workflow_run_model.webhook_callback_url,
webhook_failure_reason=workflow_run_model.webhook_failure_reason,
diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.py
index 6986e63c9..cd1c11699 100644
--- a/skyvern/forge/sdk/routes/agent_protocol.py
+++ b/skyvern/forge/sdk/routes/agent_protocol.py
@@ -3582,30 +3582,6 @@ async def _continue_workflow_run(workflow_run_id: str, organization_id: str) ->
await app.WORKFLOW_SERVICE.mark_workflow_run_as_running(workflow_run_id)
-def _workflow_request_body_from_existing_run(
- workflow_run: WorkflowRun,
- parameters: dict[str, Any] | None = None,
- run_metadata: dict[str, str] | None = None,
-) -> WorkflowRequestBody:
- return WorkflowRequestBody(
- data=parameters,
- proxy_location=workflow_run.proxy_location,
- webhook_callback_url=workflow_run.webhook_callback_url,
- totp_verification_url=workflow_run.totp_verification_url,
- totp_identifier=workflow_run.totp_identifier,
- browser_session_id=workflow_run.browser_session_id,
- browser_profile_id=workflow_run.browser_profile_id,
- max_screenshot_scrolls=workflow_run.max_screenshot_scrolls,
- max_elapsed_time_minutes=getattr(workflow_run, "max_elapsed_time_minutes", None),
- extra_http_headers=workflow_run.extra_http_headers,
- cdp_connect_headers=workflow_run.cdp_connect_headers,
- browser_address=workflow_run.browser_address,
- run_with=workflow_run.run_with,
- ai_fallback=workflow_run.ai_fallback,
- run_metadata=run_metadata,
- )
-
-
def _workflow_run_request_from_workflow_request(
*,
workflow_id: str,
@@ -3736,7 +3712,7 @@ async def retry_workflow_run(
organization_id=current_org.organization_id,
exc_info=True,
)
- legacy_workflow_request = _workflow_request_body_from_existing_run(
+ legacy_workflow_request = workflow_service.workflow_request_body_from_existing_run(
workflow_run=original_workflow_run,
parameters=original_workflow_run_parameters,
run_metadata=original_run_metadata,
diff --git a/skyvern/forge/sdk/workflow/context_manager.py b/skyvern/forge/sdk/workflow/context_manager.py
index f42c94de8..9c5a79d75 100644
--- a/skyvern/forge/sdk/workflow/context_manager.py
+++ b/skyvern/forge/sdk/workflow/context_manager.py
@@ -722,7 +722,7 @@ class WorkflowRunContext:
LOG.info("Fetching credential parameter value", parameter_key=parameter.key)
credential_id = None
- if parameter.credential_ids:
+ if parameter.credential_ids or parameter.fallback_credential_ids:
credential_id = await self.resolve_credential_parameter_id(parameter, organization.organization_id)
elif parameter.credential_id:
if self.has_parameter(parameter.credential_id) and self.has_value(parameter.credential_id):
@@ -745,8 +745,18 @@ class WorkflowRunContext:
if cached:
return cached
if not parameter.credential_ids:
- self.resolved_credential_parameter_ids[parameter.key] = parameter.credential_id
- return parameter.credential_id
+ credential_id = parameter.credential_id
+ if parameter.fallback_credential_ids:
+ selected = await app.DATABASE.workflow_run_credential_selections.get_selection(
+ workflow_run_id=self.workflow_run_id,
+ parameter_key=parameter.key,
+ )
+ if selected:
+ credential_id = selected
+ elif self.has_parameter(credential_id) and self.has_value(credential_id):
+ credential_id = self.values[credential_id]
+ self.resolved_credential_parameter_ids[parameter.key] = credential_id
+ return credential_id
credential_id = await select_credential_for_run(
workflow_run_id=self.workflow_run_id,
organization_id=organization_id,
diff --git a/skyvern/forge/sdk/workflow/credential_fallback.py b/skyvern/forge/sdk/workflow/credential_fallback.py
new file mode 100644
index 000000000..c7d5b8f5d
--- /dev/null
+++ b/skyvern/forge/sdk/workflow/credential_fallback.py
@@ -0,0 +1,255 @@
+from __future__ import annotations
+
+import structlog
+from sqlalchemy.exc import IntegrityError
+
+from skyvern.exceptions import WorkflowNotFound
+from skyvern.forge import app
+from skyvern.forge.failure_classifier import classify_from_failure_reason
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
+
+LOG = structlog.get_logger()
+
+CREDENTIAL_FAILURES = "credential_failures"
+ANY_FAILURE = "any_failure"
+VALID_FALLBACK_TRIGGERS = frozenset({CREDENTIAL_FAILURES, ANY_FAILURE})
+CREDENTIAL_FAILURE_CATEGORIES = frozenset({"AUTH_FAILURE", "CREDENTIAL_ERROR"})
+CREDENTIAL_FALLBACK_RETRY_FLAG = "CREDENTIAL_FALLBACK_RETRY"
+
+
+async def _retry_enabled_for_organization(organization_id: str, workflow_run: WorkflowRun) -> bool:
+ """Gate the auto-retry rollout. Fails closed: a retry run costs credits and re-fires webhooks.
+
+ distinct_id = organization_id (deliberately NOT run-level). backend.md's per-run
+ randomization convention serves per-request experiments; retry behavior must be
+ deterministic per customer, so a given org is fully in-or-out rather than retrying
+ on a random subset of its runs. Do not "fix" this to workflow_run_id.
+ """
+ provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)
+ if not provider:
+ return False
+ try:
+ return bool(
+ await provider.is_feature_enabled_cached(
+ CREDENTIAL_FALLBACK_RETRY_FLAG,
+ organization_id,
+ properties={
+ "organization_id": organization_id,
+ "workflow_permanent_id": workflow_run.workflow_permanent_id,
+ },
+ )
+ )
+ except Exception:
+ LOG.warning(
+ "credential_fallback.flag_error",
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ return False
+
+
+def _failure_category_names(workflow_run: WorkflowRun) -> set[str]:
+ categories = workflow_run.failure_category
+ if not categories:
+ categories = classify_from_failure_reason(workflow_run.failure_reason)
+ return {
+ category
+ for entry in categories or []
+ if isinstance(entry, dict) and isinstance(category := entry.get("category"), str)
+ }
+
+
+def _trigger_matches(workflow_run: WorkflowRun, trigger: str | None) -> bool:
+ if workflow_run.status not in {WorkflowRunStatus.failed, WorkflowRunStatus.terminated}:
+ return False
+ effective_trigger = trigger or CREDENTIAL_FAILURES
+ if effective_trigger == ANY_FAILURE:
+ return True
+ if effective_trigger != CREDENTIAL_FAILURES:
+ return False
+ return bool(_failure_category_names(workflow_run) & CREDENTIAL_FAILURE_CATEGORIES)
+
+
+async def maybe_start_credential_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:
+ try:
+ return await _maybe_start_credential_fallback_retry(workflow_run, organization_id)
+ except Exception:
+ LOG.warning(
+ "Failed to start credential fallback retry",
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+ return None
+
+
+async def _maybe_start_credential_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:
+ if workflow_run.parent_workflow_run_id or workflow_run.debug_session_id:
+ return None
+
+ if not await _retry_enabled_for_organization(organization_id, workflow_run):
+ return None
+
+ is_block_scoped_run = await app.AGENT_FUNCTION.is_block_scoped_workflow_run(workflow_run)
+ if not is_block_scoped_run:
+ is_block_scoped_run = await app.DATABASE.debug.has_block_run_for_workflow_run(
+ organization_id=organization_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ )
+ if is_block_scoped_run:
+ return None
+
+ try:
+ workflow = await app.WORKFLOW_SERVICE.get_workflow(
+ workflow_id=workflow_run.workflow_id,
+ organization_id=organization_id,
+ )
+ except WorkflowNotFound:
+ return None
+ if workflow is None:
+ return None
+
+ credential_parameters = [
+ parameter
+ for parameter in workflow.workflow_definition.parameters
+ if isinstance(parameter, CredentialParameter) and bool(parameter.fallback_credential_ids)
+ ]
+ if not credential_parameters:
+ return None
+
+ existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization_id,
+ )
+ if existing_retry_run_id:
+ return existing_retry_run_id
+
+ prior_selections = await app.DATABASE.workflow_run_credential_selections.get_selections_for_run(
+ workflow_run.workflow_run_id
+ )
+
+ next_attempt = (workflow_run.fallback_attempt or 0) + 1
+ triggered_parameters = [
+ parameter for parameter in credential_parameters if _trigger_matches(workflow_run, parameter.fallback_trigger)
+ ]
+ if not triggered_parameters:
+ return None
+
+ all_fallback_ids = list(
+ dict.fromkeys(
+ credential_id
+ for parameter in triggered_parameters
+ for credential_id in parameter.fallback_credential_ids or []
+ )
+ )
+ existing_credentials = await app.DATABASE.credentials.get_credentials_by_ids(
+ all_fallback_ids, organization_id=organization_id
+ )
+ found_credential_ids = {credential.credential_id for credential in existing_credentials}
+ missing_credential_ids = [
+ credential_id for credential_id in all_fallback_ids if credential_id not in found_credential_ids
+ ]
+ if missing_credential_ids:
+ LOG.warning(
+ "Skipping credential fallback candidates whose credentials no longer exist",
+ workflow_run_id=workflow_run.workflow_run_id,
+ missing_credential_ids=missing_credential_ids,
+ )
+
+ overrides: dict[str, str] = {}
+ for parameter in triggered_parameters:
+ fallback_ids = parameter.fallback_credential_ids or []
+ prior = prior_selections.get(parameter.key)
+ # A prior selection only marks fallback progress on a fallback retry; on an original
+ # run it may come from rotation and coincidentally appear in the fallback list.
+ base_index = fallback_ids.index(prior) + 1 if workflow_run.fallback_attempt and prior in fallback_ids else 0
+ candidate = next(
+ (
+ credential_id
+ for credential_id in fallback_ids[base_index:]
+ if credential_id in found_credential_ids and credential_id != prior
+ ),
+ None,
+ )
+ if candidate is not None:
+ overrides[parameter.key] = candidate
+ if not overrides:
+ return None
+
+ advanced_parameter_keys = list(overrides)
+ credential_parameter_keys = {
+ parameter.key
+ for parameter in workflow.workflow_definition.parameters
+ if isinstance(parameter, CredentialParameter)
+ }
+ for parameter_key, credential_id in prior_selections.items():
+ if parameter_key in credential_parameter_keys and parameter_key not in overrides:
+ overrides[parameter_key] = credential_id
+
+ parameter_tuples = await app.DATABASE.workflow_runs.get_workflow_run_parameters(
+ workflow_run_id=workflow_run.workflow_run_id,
+ )
+ original_parameters = {
+ workflow_parameter.key: workflow_run_parameter.value
+ for workflow_parameter, workflow_run_parameter in parameter_tuples
+ }
+ retry_parameters = {**original_parameters, **overrides}
+
+ organization = await app.DATABASE.organizations.get_organization(organization_id)
+ if organization is None:
+ return None
+
+ from skyvern.services.workflow_service import run_workflow, workflow_request_body_from_existing_run
+
+ workflow_request = workflow_request_body_from_existing_run(
+ workflow_run=workflow_run,
+ parameters=retry_parameters,
+ )
+ workflow_request.browser_session_id = None
+ workflow_request.browser_profile_id = None
+ workflow_request.extra_http_headers = app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers(
+ workflow_request.extra_http_headers
+ )
+ try:
+ # A fresh context keeps run creation (and the background execution task it spawns,
+ # which inherits contextvars) from attributing the retry to the failed run.
+ with skyvern_context.scoped(SkyvernContext()):
+ retry_run = await run_workflow(
+ workflow_id=workflow_run.workflow_permanent_id,
+ organization=organization,
+ workflow_request=workflow_request,
+ template=False,
+ version=workflow.version,
+ trigger_type=workflow_run.trigger_type,
+ workflow_schedule_id=workflow_run.workflow_schedule_id,
+ retried_from_workflow_run_id=workflow_run.workflow_run_id,
+ fallback_attempt=next_attempt,
+ ignore_inherited_workflow_system_prompt=workflow_run.ignore_inherited_workflow_system_prompt,
+ )
+ except IntegrityError as exc:
+ error_text = str(exc.orig) if exc.orig is not None else str(exc)
+ if (
+ "ix_workflow_runs_retried_from_workflow_run_id" not in error_text
+ and "retried_from_workflow_run_id" not in error_text
+ ):
+ raise
+ existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=organization_id,
+ )
+ if not existing_retry_run_id:
+ raise
+ return existing_retry_run_id
+
+ LOG.info(
+ "Started credential fallback retry",
+ workflow_run_id=workflow_run.workflow_run_id,
+ new_run_id=retry_run.workflow_run_id,
+ fallback_attempt=next_attempt,
+ advanced_parameter_keys=advanced_parameter_keys,
+ )
+ return retry_run.workflow_run_id
diff --git a/skyvern/forge/sdk/workflow/models/parameter.py b/skyvern/forge/sdk/workflow/models/parameter.py
index 0ce962523..d7b83bbd1 100644
--- a/skyvern/forge/sdk/workflow/models/parameter.py
+++ b/skyvern/forge/sdk/workflow/models/parameter.py
@@ -127,6 +127,8 @@ class CredentialParameter(Parameter):
credential_id: str
credential_ids: list[str] | None = None
selection_strategy: str | None = None
+ fallback_credential_ids: list[str] | None = None
+ fallback_trigger: str | None = None
created_at: datetime
modified_at: datetime
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index ee5d91399..eda399209 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -239,6 +239,8 @@ class WorkflowRun(BaseModel):
totp_identifier: str | None = None
failure_reason: str | None = None
failure_category: list[dict[str, Any]] | None = None
+ retried_from_workflow_run_id: str | None = None
+ fallback_attempt: int | None = None
parent_workflow_run_id: str | None = None
workflow_title: str | None = None
max_screenshot_scrolls: int | None = None
@@ -346,6 +348,8 @@ class WorkflowRunResponseBase(BaseModel):
status: WorkflowRunStatus
failure_reason: str | None = None
failure_category: list[dict[str, Any]] | None = None
+ retried_from_workflow_run_id: str | None = None
+ retried_by_workflow_run_id: str | None = None
proxy_location: ProxyLocationInput = None
webhook_callback_url: str | None = None
webhook_failure_reason: str | None = None
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index d2cc3dbe7..92a34b7b8 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -83,6 +83,10 @@ from skyvern.forge.sdk.workflow.browser_profile_key import (
build_workflow_browser_session_storage_key,
render_browser_profile_key,
)
+from skyvern.forge.sdk.workflow.credential_fallback import (
+ VALID_FALLBACK_TRIGGERS,
+ maybe_start_credential_fallback_retry,
+)
from skyvern.forge.sdk.workflow.credential_selection import (
VALID_SELECTION_STRATEGIES,
normalize_selection_strategy,
@@ -732,6 +736,23 @@ class WorkflowService:
exc_info=True,
)
+ @staticmethod
+ async def _start_credential_fallback_retry_best_effort(workflow_run: WorkflowRun) -> None:
+ try:
+ await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)
+ except Exception:
+ LOG.warning(
+ "Credential fallback retry hook failed",
+ workflow_run_id=workflow_run.workflow_run_id,
+ organization_id=workflow_run.organization_id,
+ exc_info=True,
+ )
+
+ def _schedule_credential_fallback_retry(self, workflow_run: WorkflowRun) -> None:
+ task = asyncio.create_task(self._start_credential_fallback_retry_best_effort(workflow_run))
+ self._background_tasks.add(task)
+ task.add_done_callback(self._background_tasks.discard)
+
@staticmethod
def _determine_cache_invalidation(
previous_blocks: list[dict[str, Any]],
@@ -1138,28 +1159,67 @@ class WorkflowService:
) -> None:
credential_ids_to_validate: list[str] = []
for parameter in parameters:
- credential_ids = getattr(parameter, "credential_ids", None)
- if credential_ids is None:
- continue
key = getattr(parameter, "key", None) or "<unknown>"
- if not credential_ids:
+ credential_ids = getattr(parameter, "credential_ids", None)
+ if credential_ids is not None:
+ if not credential_ids:
+ raise SkyvernHTTPException(
+ message=f"credential_ids for credential parameter {key} must be non-empty.",
+ status_code=400,
+ )
+ credential_ids = list(dict.fromkeys(credential_ids))
+ parameter.credential_ids = credential_ids
+ selection_strategy = getattr(parameter, "selection_strategy", None)
+ if normalize_selection_strategy(selection_strategy) not in VALID_SELECTION_STRATEGIES:
+ raise SkyvernHTTPException(
+ message=(
+ f"selection_strategy for credential parameter {key} must be one of: "
+ f"{', '.join(sorted(VALID_SELECTION_STRATEGIES))}."
+ ),
+ status_code=400,
+ )
+ parameter.credential_id = credential_ids[0]
+ credential_ids_to_validate.extend(credential_ids)
+
+ fallback_credential_ids = getattr(parameter, "fallback_credential_ids", None)
+ if fallback_credential_ids is not None:
+ fallback_credential_ids = list(dict.fromkeys(fallback_credential_ids))
+ if credential_ids is None:
+ primary_credential_id = getattr(parameter, "credential_id", None)
+ fallback_credential_ids = [
+ credential_id
+ for credential_id in fallback_credential_ids
+ if credential_id != primary_credential_id
+ ]
+ parameter.fallback_credential_ids = fallback_credential_ids or None
+ credential_ids_to_validate.extend(fallback_credential_ids)
+
+ if (
+ getattr(parameter, "credential_ids", None) is not None
+ and getattr(parameter, "fallback_credential_ids", None) is not None
+ ):
raise SkyvernHTTPException(
- message=f"credential_ids for credential parameter {key} must be non-empty.",
+ message=(
+ f"credential parameter {key} cannot combine credential_ids rotation with "
+ "fallback_credential_ids; configure one or the other."
+ ),
status_code=400,
)
- credential_ids = list(dict.fromkeys(credential_ids))
- parameter.credential_ids = credential_ids
- selection_strategy = getattr(parameter, "selection_strategy", None)
- if normalize_selection_strategy(selection_strategy) not in VALID_SELECTION_STRATEGIES:
+
+ fallback_trigger = getattr(parameter, "fallback_trigger", None)
+ if fallback_trigger is not None and fallback_trigger not in VALID_FALLBACK_TRIGGERS:
raise SkyvernHTTPException(
message=(
- f"selection_strategy for credential parameter {key} must be one of: "
- f"{', '.join(sorted(VALID_SELECTION_STRATEGIES))}."
+ f"fallback_trigger for credential parameter {key} must be one of: "
+ f"{', '.join(sorted(VALID_FALLBACK_TRIGGERS))}."
),
status_code=400,
)
- parameter.credential_id = credential_ids[0]
- credential_ids_to_validate.extend(credential_ids)
+ if fallback_trigger is not None and not getattr(parameter, "fallback_credential_ids", None):
+ raise SkyvernHTTPException(
+ message=f"fallback_trigger for credential parameter {key} requires fallback_credential_ids.",
+ status_code=400,
+ )
await self._validate_credential_ids(credential_ids_to_validate, organization)
@@ -1174,6 +1234,18 @@ class WorkflowService:
if isinstance(parameter, CredentialParameter) and bool(parameter.credential_ids)
]
+ @staticmethod
+ def _get_credential_parameters_with_configured_selection(workflow: Workflow) -> list[CredentialParameter]:
+ workflow_definition = getattr(workflow, "workflow_definition", None)
+ if workflow_definition is None:
+ return []
+ return [
+ parameter
+ for parameter in workflow_definition.parameters
+ if isinstance(parameter, CredentialParameter)
+ and (bool(parameter.credential_ids) or bool(parameter.fallback_credential_ids))
+ ]
+
def _get_run_credential_parameter_overrides(
self,
*,
@@ -1184,7 +1256,7 @@ class WorkflowService:
return {}
overrides: dict[str, str] = {}
- for parameter in self._get_rotating_credential_parameters(workflow):
+ for parameter in self._get_credential_parameters_with_configured_selection(workflow):
if parameter.key not in request_data:
continue
@@ -1194,12 +1266,18 @@ class WorkflowService:
if not isinstance(override, str):
raise InvalidCredentialId(f"<non-string value of type {type(override).__name__}>")
- credential_ids = parameter.credential_ids or []
+ credential_ids = list(
+ dict.fromkeys(
+ (parameter.credential_ids or [])
+ + (parameter.fallback_credential_ids or [])
+ + [parameter.credential_id]
+ )
+ )
if override not in credential_ids:
raise SkyvernHTTPException(
message=(
f"Credential override for parameter {parameter.key} must be one of the configured "
- "rotation credentials."
+ "rotation or fallback credentials."
),
status_code=400,
)
@@ -1370,6 +1448,8 @@ class WorkflowService:
workflow_run_id: str | None = None,
trigger_type: WorkflowRunTriggerType | None = None,
workflow_schedule_id: str | None = None,
+ retried_from_workflow_run_id: str | None = None,
+ fallback_attempt: int | None = None,
ignore_inherited_workflow_system_prompt: bool = False,
copilot_session_id: str | None = None,
resolved_workflow_id: str | None = None,
@@ -1478,6 +1558,8 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
trigger_type=resolved_trigger_type,
workflow_schedule_id=workflow_schedule_id,
+ retried_from_workflow_run_id=retried_from_workflow_run_id,
+ fallback_attempt=fallback_attempt,
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
copilot_session_id=resolved_copilot_session_id,
)
@@ -4542,21 +4624,29 @@ class WorkflowService:
organization_id: str | None,
workflow_permanent_id: str | None,
) -> str:
- if not parameter.credential_ids or not workflow_run_id or not organization_id or not workflow_permanent_id:
+ if not workflow_run_id or not organization_id or not workflow_permanent_id:
return parameter.credential_id
workflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.workflow_run_contexts.get(workflow_run_id)
if workflow_run_context:
return await workflow_run_context.resolve_credential_parameter_id(parameter, organization_id)
- return await select_credential_for_run(
- workflow_run_id=workflow_run_id,
- organization_id=organization_id,
- workflow_permanent_id=workflow_permanent_id,
- parameter_key=parameter.key,
- credential_ids=parameter.credential_ids,
- selection_strategy=parameter.selection_strategy,
- )
+ if parameter.credential_ids:
+ return await select_credential_for_run(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_permanent_id,
+ parameter_key=parameter.key,
+ credential_ids=parameter.credential_ids,
+ selection_strategy=parameter.selection_strategy,
+ )
+ if parameter.fallback_credential_ids:
+ selected = await app.DATABASE.workflow_run_credential_selections.get_selection(
+ workflow_run_id=workflow_run_id,
+ parameter_key=parameter.key,
+ )
+ return selected or parameter.credential_id
+ return parameter.credential_id
async def _apply_login_block_credential_proxy_pin(
self,
@@ -5874,6 +5964,8 @@ class WorkflowService:
workflow_run_id: str | None = None,
trigger_type: WorkflowRunTriggerType | None = None,
workflow_schedule_id: str | None = None,
+ retried_from_workflow_run_id: str | None = None,
+ fallback_attempt: int | None = None,
ignore_inherited_workflow_system_prompt: bool = False,
copilot_session_id: str | None = None,
) -> WorkflowRun:
@@ -5960,6 +6052,8 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
trigger_type=trigger_type,
workflow_schedule_id=workflow_schedule_id,
+ retried_from_workflow_run_id=retried_from_workflow_run_id,
+ fallback_attempt=fallback_attempt,
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
copilot_session_id=copilot_session_id,
)
@@ -6074,6 +6168,8 @@ class WorkflowService:
workflow_run_id=workflow_run_id,
trigger_type=trigger_type,
workflow_schedule_id=workflow_schedule_id,
+ retried_from_workflow_run_id=retried_from_workflow_run_id,
+ fallback_attempt=fallback_attempt,
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
copilot_session_id=copilot_session_id,
)
@@ -6296,13 +6392,15 @@ class WorkflowService:
failure_category_source=failure_category_source,
)
- return await self._update_workflow_run_status(
+ workflow_run = await self._update_workflow_run_status(
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.failed,
failure_reason=failure_reason,
run_with=run_with,
failure_category=failure_category,
)
+ self._schedule_credential_fallback_retry(workflow_run)
+ return workflow_run
async def mark_workflow_run_as_running(self, workflow_run_id: str, run_with: str | None = None) -> WorkflowRun:
# Conditional UPDATE refuses to resurrect a finalized wr — prevents the
@@ -6382,13 +6480,15 @@ class WorkflowService:
failure_category_source=failure_category_source,
)
- return await self._update_workflow_run_status(
+ workflow_run = await self._update_workflow_run_status(
workflow_run_id=workflow_run_id,
status=WorkflowRunStatus.terminated,
failure_reason=failure_reason,
run_with=run_with,
failure_category=failure_category,
)
+ self._schedule_credential_fallback_retry(workflow_run)
+ return workflow_run
async def mark_workflow_run_as_canceled(self, workflow_run_id: str) -> WorkflowRun:
"""Cancel a workflow run, rejecting the transition if the run has already
@@ -6998,6 +7098,7 @@ class WorkflowService:
output_parameter_tuples,
(recording_urls, recording_archived),
(downloaded_files, downloaded_file_urls),
+ retried_by_workflow_run_id,
) = await asyncio.gather(
self.get_recent_workflow_screenshot_urls(
workflow_run_id=workflow_run_id,
@@ -7009,6 +7110,10 @@ class WorkflowService:
),
self._fetch_recording_urls(workflow_run, task_v2, organization_id),
self._fetch_downloaded_files(workflow_run, task_v2),
+ app.DATABASE.workflow_runs.get_workflow_run_retried_by(
+ workflow_run_id=workflow_run_id,
+ organization_id=workflow_run.organization_id,
+ ),
)
screenshot_urls: list[str] | None = screenshot_urls_raw or None
# Preserve legacy singular contract: last element is the newest.
@@ -7069,6 +7174,8 @@ class WorkflowService:
status=workflow_run.status,
failure_reason=workflow_run.failure_reason,
failure_category=workflow_run.failure_category,
+ retried_from_workflow_run_id=workflow_run.retried_from_workflow_run_id,
+ retried_by_workflow_run_id=retried_by_workflow_run_id,
proxy_location=workflow_run.proxy_location,
webhook_callback_url=workflow_run.webhook_callback_url,
webhook_failure_reason=workflow_run.webhook_failure_reason,
diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
index 13effc94d..8aa9cd742 100644
--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py
+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
@@ -172,6 +172,8 @@ def convert_workflow_definition(
credential_id=parameter.credential_id,
credential_ids=parameter.credential_ids,
selection_strategy=parameter.selection_strategy,
+ fallback_credential_ids=parameter.fallback_credential_ids,
+ fallback_trigger=parameter.fallback_trigger,
created_at=now,
modified_at=now,
)
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index 093f94123..f5ddf6fa4 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -617,6 +617,8 @@ class CredentialParameterYAML(ParameterYAML):
credential_id: str
credential_ids: list[str] | None = None
selection_strategy: str | None = None
+ fallback_credential_ids: list[str] | None = None
+ fallback_trigger: str | None = None
class BitwardenSensitiveInformationParameterYAML(ParameterYAML):
diff --git a/skyvern/services/workflow_service.py b/skyvern/services/workflow_service.py
index 551cc1f73..047c2de7b 100644
--- a/skyvern/services/workflow_service.py
+++ b/skyvern/services/workflow_service.py
@@ -16,6 +16,30 @@ from skyvern.schemas.runs import RunStatus, RunType, WorkflowRunRequest, Workflo
LOG = structlog.get_logger(__name__)
+def workflow_request_body_from_existing_run(
+ workflow_run: WorkflowRun,
+ parameters: dict[str, t.Any] | None = None,
+ run_metadata: dict[str, str] | None = None,
+) -> WorkflowRequestBody:
+ return WorkflowRequestBody(
+ data=parameters,
+ proxy_location=workflow_run.proxy_location,
+ webhook_callback_url=workflow_run.webhook_callback_url,
+ totp_verification_url=workflow_run.totp_verification_url,
+ totp_identifier=workflow_run.totp_identifier,
+ browser_session_id=workflow_run.browser_session_id,
+ browser_profile_id=workflow_run.browser_profile_id,
+ max_screenshot_scrolls=workflow_run.max_screenshot_scrolls,
+ max_elapsed_time_minutes=workflow_run.max_elapsed_time_minutes,
+ extra_http_headers=workflow_run.extra_http_headers,
+ cdp_connect_headers=workflow_run.cdp_connect_headers,
+ browser_address=workflow_run.browser_address,
+ run_with=workflow_run.run_with,
+ ai_fallback=workflow_run.ai_fallback,
+ run_metadata=run_metadata,
+ )
+
+
async def prepare_workflow(
workflow_id: str,
organization: Organization,
@@ -30,6 +54,8 @@ async def prepare_workflow(
trigger_type: WorkflowRunTriggerType | None = None,
workflow_schedule_id: str | None = None,
workflow_run_id: str | None = None,
+ retried_from_workflow_run_id: str | None = None,
+ fallback_attempt: int | None = None,
ignore_inherited_workflow_system_prompt: bool = False,
copilot_session_id: str | None = None,
resolved_workflow_id: str | None = None,
@@ -59,6 +85,8 @@ async def prepare_workflow(
parent_workflow_run_id=parent_workflow_run_id,
trigger_type=trigger_type,
workflow_schedule_id=workflow_schedule_id,
+ retried_from_workflow_run_id=retried_from_workflow_run_id,
+ fallback_attempt=fallback_attempt,
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
copilot_session_id=copilot_session_id,
resolved_workflow_id=resolved_workflow_id,
@@ -110,6 +138,8 @@ async def run_workflow(
parent_workflow_run_id: str | None = None,
trigger_type: WorkflowRunTriggerType | None = None,
workflow_schedule_id: str | None = None,
+ retried_from_workflow_run_id: str | None = None,
+ fallback_attempt: int | None = None,
ignore_inherited_workflow_system_prompt: bool = False,
tag_write_context: TagWriteContext | None = None,
) -> WorkflowRun:
@@ -124,6 +154,8 @@ async def run_workflow(
parent_workflow_run_id=parent_workflow_run_id,
trigger_type=trigger_type,
workflow_schedule_id=workflow_schedule_id,
+ retried_from_workflow_run_id=retried_from_workflow_run_id,
+ fallback_attempt=fallback_attempt,
ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
tag_write_context=tag_write_context,
)
diff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.py
new file mode 100644
index 000000000..0cc180389
--- /dev/null
+++ b/tests/unit/test_credential_fallback.py
@@ -0,0 +1,767 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from sqlalchemy.exc import IntegrityError
+
+from skyvern.exceptions import InvalidCredentialId, SkyvernHTTPException
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+from skyvern.forge.sdk.workflow.credential_fallback import (
+ ANY_FAILURE,
+ CREDENTIAL_FAILURES,
+ _trigger_matches,
+ maybe_start_credential_fallback_retry,
+)
+from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter
+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus
+from skyvern.forge.sdk.workflow.service import WorkflowService
+from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition
+from skyvern.schemas.workflows import CredentialParameterYAML, WorkflowDefinitionYAML
+
+
+def _credential_parameter(
+ *,
+ key: str = "login_cred",
+ credential_id: str = "cred_primary",
+ credential_ids: list[str] | None = None,
+ fallback_credential_ids: list[str] | None = None,
+ fallback_trigger: str | None = None,
+) -> CredentialParameter:
+ now = datetime.now(timezone.utc)
+ return CredentialParameter(
+ key=key,
+ credential_parameter_id=f"cp_{key}",
+ workflow_id="wf_test",
+ credential_id=credential_id,
+ credential_ids=credential_ids,
+ fallback_credential_ids=fallback_credential_ids,
+ fallback_trigger=fallback_trigger,
+ created_at=now,
+ modified_at=now,
+ )
+
+
+def _workflow_run(
+ *,
+ workflow_run_id: str = "wr_failed",
+ status: WorkflowRunStatus = WorkflowRunStatus.failed,
+ failure_category: list[dict] | None = None,
+ failure_reason: str | None = "login failed",
+ fallback_attempt: int | None = None,
+ parent_workflow_run_id: str | None = None,
+ debug_session_id: str | None = None,
+ retried_from_workflow_run_id: str | None = None,
+ browser_session_id: str | None = None,
+ browser_profile_id: str | None = None,
+ extra_http_headers: dict[str, str] | None = None,
+ workflow_schedule_id: str | None = None,
+) -> WorkflowRun:
+ now = datetime.now(timezone.utc)
+ return WorkflowRun(
+ workflow_run_id=workflow_run_id,
+ workflow_id="wf_test",
+ workflow_permanent_id="wpid_test",
+ organization_id="org_test",
+ status=status,
+ failure_reason=failure_reason,
+ failure_category=failure_category,
+ fallback_attempt=fallback_attempt,
+ parent_workflow_run_id=parent_workflow_run_id,
+ debug_session_id=debug_session_id,
+ retried_from_workflow_run_id=retried_from_workflow_run_id,
+ browser_session_id=browser_session_id,
+ browser_profile_id=browser_profile_id,
+ extra_http_headers=extra_http_headers,
+ workflow_schedule_id=workflow_schedule_id,
+ trigger_type=WorkflowRunTriggerType.api,
+ created_at=now,
+ modified_at=now,
+ )
+
+
+def _workflow(parameters: list[CredentialParameter]) -> SimpleNamespace:
+ return SimpleNamespace(
+ workflow_id="wf_test",
+ workflow_permanent_id="wpid_test",
+ version=7,
+ title="Workflow",
+ workflow_definition=SimpleNamespace(parameters=parameters),
+ )
+
+
+async def _validate_parameter(parameter: CredentialParameter, existing_ids: list[str] | None = None) -> None:
+ service = WorkflowService()
+ organization = SimpleNamespace(organization_id="org_test")
+ existing = [SimpleNamespace(credential_id=credential_id) for credential_id in existing_ids or []]
+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
+ mock_app.DATABASE.credentials.get_credentials_by_ids = AsyncMock(return_value=existing)
+ await service._validate_and_normalize_credential_rotation_parameters([parameter], organization)
+
+
+async def test_workflow_save_validation_dedupes_fallback_credentials_preserving_order() -> None:
+ parameter = _credential_parameter(
+ fallback_credential_ids=["cred_b", "cred_c", "cred_b", "cred_d", "cred_c"],
+ )
+
+ await _validate_parameter(parameter, ["cred_b", "cred_c", "cred_d"])
+
+ assert parameter.fallback_credential_ids == ["cred_b", "cred_c", "cred_d"]
+ assert parameter.credential_id == "cred_primary"
+
+
+async def test_workflow_save_validation_rejects_unknown_fallback_credential_id() -> None:
+ parameter = _credential_parameter(fallback_credential_ids=["cred_missing"])
+
+ with pytest.raises(InvalidCredentialId):
+ await _validate_parameter(parameter, [])
+
+
+async def test_workflow_save_validation_rejects_bad_fallback_trigger() -> None:
+ parameter = _credential_parameter(fallback_credential_ids=["cred_b"], fallback_trigger="later")
+
+ with pytest.raises(SkyvernHTTPException, match="fallback_trigger"):
+ await _validate_parameter(parameter, ["cred_b"])
+
+
+async def test_workflow_save_validation_rejects_trigger_without_fallback_list() -> None:
+ parameter = _credential_parameter(fallback_trigger=ANY_FAILURE)
+
+ with pytest.raises(SkyvernHTTPException, match="requires fallback_credential_ids"):
+ await _validate_parameter(parameter)
+
+
+async def test_workflow_save_validation_normalizes_empty_fallback_list_to_none() -> None:
+ parameter = _credential_parameter(fallback_credential_ids=[])
+
+ await _validate_parameter(parameter)
+
+ assert parameter.fallback_credential_ids is None
+
+
+async def test_workflow_save_validation_drops_primary_credential_from_fallbacks() -> None:
+ parameter = _credential_parameter(fallback_credential_ids=["cred_primary", "cred_b"])
+
+ await _validate_parameter(parameter, ["cred_b"])
+
+ assert parameter.fallback_credential_ids == ["cred_b"]
+
+
+async def test_workflow_save_validation_rejects_trigger_when_fallbacks_collapse_to_primary() -> None:
+ parameter = _credential_parameter(fallback_credential_ids=["cred_primary"], fallback_trigger=ANY_FAILURE)
+
+ with pytest.raises(SkyvernHTTPException, match="requires fallback_credential_ids"):
+ await _validate_parameter(parameter)
+
+
+async def test_workflow_save_validation_rejects_rotation_with_fallback_credentials() -> None:
+ parameter = _credential_parameter(
+ credential_ids=["cred_primary", "cred_alt"],
+ fallback_credential_ids=["cred_primary", "cred_b"],
+ )
+
+ with pytest.raises(SkyvernHTTPException) as exc_info:
+ await _validate_parameter(parameter, ["cred_primary", "cred_alt", "cred_b"])
+
+ assert exc_info.value.status_code == 400
+ assert exc_info.value.message == (
+ "credential parameter login_cred cannot combine credential_ids rotation with fallback_credential_ids; "
+ "configure one or the other."
+ )
+
+
+async def test_workflow_save_validation_allows_rotation_without_fallback_credentials() -> None:
+ parameter = _credential_parameter(credential_ids=["cred_primary", "cred_alt"])
+
+ await _validate_parameter(parameter, ["cred_primary", "cred_alt"])
+
+ assert parameter.credential_ids == ["cred_primary", "cred_alt"]
+ assert parameter.credential_id == "cred_primary"
+
+
+async def test_workflow_save_validation_allows_fallback_credentials_without_rotation() -> None:
+ parameter = _credential_parameter(fallback_credential_ids=["cred_b", "cred_c"])
+
+ await _validate_parameter(parameter, ["cred_b", "cred_c"])
+
+ assert parameter.credential_ids is None
+ assert parameter.fallback_credential_ids == ["cred_b", "cred_c"]
+
+
[email protected]("category", ["AUTH_FAILURE", "CREDENTIAL_ERROR"])
+def test_credential_failure_trigger_matches_credential_categories(category: str) -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": category}])
+
+ assert _trigger_matches(workflow_run, CREDENTIAL_FAILURES)
+
+
+def test_credential_failure_trigger_ignores_noncredential_categories() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "ELEMENT_NOT_FOUND"}])
+
+ assert not _trigger_matches(workflow_run, CREDENTIAL_FAILURES)
+
+
+def test_credential_failure_trigger_falls_back_to_failure_reason_classification() -> None:
+ workflow_run = _workflow_run(failure_category=None, failure_reason="The password was incorrect during login")
+
+ assert _trigger_matches(workflow_run, CREDENTIAL_FAILURES)
+
+
[email protected]("status", [WorkflowRunStatus.failed, WorkflowRunStatus.terminated])
+def test_any_failure_trigger_matches_failed_and_terminated_runs(status: WorkflowRunStatus) -> None:
+ workflow_run = _workflow_run(status=status, failure_category=[{"category": "ELEMENT_NOT_FOUND"}])
+
+ assert _trigger_matches(workflow_run, ANY_FAILURE)
+
+
+PROXY_SESSION_MARKER_HEADER = "x-proxy-session-marker"
+
+
+def _strip_marker_header(extra_http_headers: dict[str, str] | None) -> dict[str, str] | None:
+ if not extra_http_headers:
+ return extra_http_headers
+ return {key: value for key, value in extra_http_headers.items() if key != PROXY_SESSION_MARKER_HEADER} or None
+
+
+async def _run_fallback_retry(
+ *,
+ workflow_run: WorkflowRun,
+ parameters: list[CredentialParameter],
+ existing_retry_run_id: str | None = None,
+ retried_by_results: list[str | None] | None = None,
+ prior_selections: dict[str, str] | None = None,
+ block_scoped: bool = False,
+ missing_credential_ids: set[str] | None = None,
+ run_workflow_error: Exception | None = None,
+ flag_enabled: bool = True,
+ flag_error: Exception | None = None,
+) -> tuple[str | None, AsyncMock, MagicMock]:
+ captured: dict[str, object] = {}
+
+ async def fake_run_workflow(**kwargs: object) -> WorkflowRun:
+ captured["context_at_call"] = skyvern_context.current()
+ if run_workflow_error is not None:
+ raise run_workflow_error
+ return _workflow_run(
+ workflow_run_id="wr_retry",
+ status=WorkflowRunStatus.created,
+ failure_category=None,
+ failure_reason=None,
+ retried_from_workflow_run_id=kwargs["retried_from_workflow_run_id"], # type: ignore[arg-type]
+ fallback_attempt=kwargs["fallback_attempt"], # type: ignore[arg-type]
+ )
+
+ run_workflow_mock = AsyncMock(side_effect=fake_run_workflow)
+ run_workflow_mock.captured = captured
+ with (
+ patch("skyvern.forge.sdk.workflow.credential_fallback.app") as mock_app,
+ patch("skyvern.services.workflow_service.run_workflow", run_workflow_mock),
+ ):
+ mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(
+ side_effect=flag_error, return_value=flag_enabled
+ )
+ mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)
+ mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = _strip_marker_header
+ mock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)
+ mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=_workflow(parameters))
+ if retried_by_results is not None:
+ mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(side_effect=retried_by_results)
+ else:
+ mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(return_value=existing_retry_run_id)
+ mock_app.DATABASE.workflow_run_credential_selections.get_selections_for_run = AsyncMock(
+ return_value=prior_selections or {}
+ )
+
+ async def fake_get_credentials_by_ids(credential_ids: list[str], organization_id: str) -> list[SimpleNamespace]:
+ return [
+ SimpleNamespace(credential_id=credential_id)
+ for credential_id in credential_ids
+ if credential_id not in (missing_credential_ids or set())
+ ]
+
+ mock_app.DATABASE.credentials.get_credentials_by_ids = AsyncMock(side_effect=fake_get_credentials_by_ids)
+ mock_app.DATABASE.workflow_runs.get_workflow_run_parameters = AsyncMock(
+ return_value=[
+ (
+ SimpleNamespace(key="account_id"),
+ SimpleNamespace(value="acct_1"),
+ )
+ ]
+ )
+ mock_app.DATABASE.organizations.get_organization = AsyncMock(
+ return_value=SimpleNamespace(organization_id="org_test")
+ )
+ result = await maybe_start_credential_fallback_retry(workflow_run, "org_test")
+ return result, run_workflow_mock, mock_app
+
+
+async def test_fallback_attempt_one_uses_first_fallback_and_preserves_other_pins() -> None:
+ workflow_run = _workflow_run(
+ failure_category=[{"category": "AUTH_FAILURE"}],
+ browser_session_id="pbs_live",
+ browser_profile_id="bprof_old",
+ workflow_schedule_id="ws_1",
+ )
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])
+ backup = _credential_parameter(
+ key="backup_cred",
+ credential_id="cred_backup_primary",
+ credential_ids=["cred_backup_primary", "cred_backup_alt"],
+ )
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login, backup],
+ prior_selections={"backup_cred": "cred_backup_alt", "login_cred": "cred_primary"},
+ )
+
+ assert result == "wr_retry"
+ kwargs = run_workflow_mock.await_args.kwargs
+ assert kwargs["retried_from_workflow_run_id"] == "wr_failed"
+ assert kwargs["fallback_attempt"] == 1
+ assert kwargs["version"] == 7
+ assert kwargs["trigger_type"] == WorkflowRunTriggerType.api
+ assert kwargs["workflow_schedule_id"] == "ws_1"
+ assert kwargs["workflow_request"].browser_session_id is None
+ assert kwargs["workflow_request"].browser_profile_id is None
+ assert kwargs["workflow_request"].data == {
+ "account_id": "acct_1",
+ "login_cred": "cred_fb1",
+ "backup_cred": "cred_backup_alt",
+ }
+
+
+async def test_fallback_attempt_two_advances_from_parameter_prior_selection() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], fallback_attempt=1)
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ prior_selections={"login_cred": "cred_fb1"},
+ )
+
+ assert result == "wr_retry"
+ assert run_workflow_mock.await_args.kwargs["fallback_attempt"] == 2
+ assert run_workflow_mock.await_args.kwargs["workflow_request"].data["login_cred"] == "cred_fb2"
+
+
+async def test_mixed_triggers_advance_each_parameter_from_its_own_prior_selection() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], fallback_attempt=1)
+ any_failure_param = _credential_parameter(
+ key="login_cred",
+ fallback_credential_ids=["cred_a_fb1", "cred_a_fb2"],
+ fallback_trigger=ANY_FAILURE,
+ )
+ credential_failure_param = _credential_parameter(
+ key="backup_cred",
+ credential_id="cred_backup_primary",
+ fallback_credential_ids=["cred_b_fb1", "cred_b_fb2"],
+ fallback_trigger=CREDENTIAL_FAILURES,
+ )
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[any_failure_param, credential_failure_param],
+ prior_selections={"login_cred": "cred_a_fb1"},
+ )
+
+ assert result == "wr_retry"
+ kwargs = run_workflow_mock.await_args.kwargs
+ assert kwargs["fallback_attempt"] == 2
+ assert kwargs["workflow_request"].data["login_cred"] == "cred_a_fb2"
+ assert kwargs["workflow_request"].data["backup_cred"] == "cred_b_fb1"
+
+
+async def test_fallback_exhaustion_creates_no_retry_run() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], fallback_attempt=2)
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ prior_selections={"login_cred": "cred_fb2"},
+ )
+
+ assert result is None
+ run_workflow_mock.assert_not_awaited()
+
+
+async def test_flag_disabled_creates_no_retry_run_and_reads_nothing() -> None:
+ """Default-off gate: a run that would otherwise retry must do nothing at all.
+
+ No retry run means no credits consumed and no webhook re-fired.
+ """
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])
+
+ result, run_workflow_mock, mock_app = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ flag_enabled=False,
+ )
+
+ assert result is None
+ run_workflow_mock.assert_not_awaited()
+ mock_app.WORKFLOW_SERVICE.get_workflow.assert_not_called()
+ mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by.assert_not_called()
+
+
+async def test_flag_evaluation_failure_fails_closed() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ flag_error=RuntimeError("posthog is down"),
+ )
+
+ assert result is None
+ run_workflow_mock.assert_not_awaited()
+
+
+async def test_deleted_fallback_credential_is_skipped_in_favor_of_next() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ missing_credential_ids={"cred_fb1"},
+ )
+
+ assert result == "wr_retry"
+ kwargs = run_workflow_mock.await_args.kwargs
+ assert kwargs["fallback_attempt"] == 1
+ assert kwargs["workflow_request"].data["login_cred"] == "cred_fb2"
+
+
+async def test_all_fallback_credentials_deleted_does_not_advance() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ missing_credential_ids={"cred_fb1", "cred_fb2"},
+ )
+
+ assert result is None
+ run_workflow_mock.assert_not_awaited()
+
+
+async def test_rotation_selected_prior_in_fallback_list_does_not_fake_progress() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(
+ credential_id="cred_a",
+ credential_ids=["cred_a", "cred_b"],
+ fallback_credential_ids=["cred_c", "cred_b"],
+ )
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ prior_selections={"login_cred": "cred_b"},
+ )
+
+ assert result == "wr_retry"
+ kwargs = run_workflow_mock.await_args.kwargs
+ assert kwargs["fallback_attempt"] == 1
+ assert kwargs["workflow_request"].data["login_cred"] == "cred_c"
+
+
+async def test_retry_request_strips_credential_proxy_session_headers() -> None:
+ workflow_run = _workflow_run(
+ failure_category=[{"category": "AUTH_FAILURE"}],
+ extra_http_headers={PROXY_SESSION_MARKER_HEADER: "proxy_sess_1", "x-custom": "keep"},
+ )
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(workflow_run=workflow_run, parameters=[login])
+
+ assert result == "wr_retry"
+ assert run_workflow_mock.await_args.kwargs["workflow_request"].extra_http_headers == {"x-custom": "keep"}
+
+
+async def test_retry_run_creation_does_not_inherit_ambient_skyvern_context() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])
+ ambient = SkyvernContext(run_id="wr_failed", workflow_run_id="wr_failed", root_workflow_run_id="wr_failed")
+ skyvern_context.set(ambient)
+ try:
+ result, run_workflow_mock, _ = await _run_fallback_retry(workflow_run=workflow_run, parameters=[login])
+ assert skyvern_context.current() is ambient
+ finally:
+ skyvern_context.reset()
+
+ assert result == "wr_retry"
+ context_at_call = run_workflow_mock.captured["context_at_call"]
+ assert context_at_call is None or (context_at_call.run_id is None and context_at_call.root_workflow_run_id is None)
+
+
+async def test_duplicate_retry_integrity_error_recovers_existing_retry_run() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])
+ duplicate_error = IntegrityError(
+ "INSERT INTO workflow_runs",
+ None,
+ Exception('duplicate key value violates unique constraint "ix_workflow_runs_retried_from_workflow_run_id"'),
+ )
+
+ result, _, mock_app = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ retried_by_results=[None, "wr_existing"],
+ run_workflow_error=duplicate_error,
+ )
+
+ assert result == "wr_existing"
+ assert mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by.await_count == 2
+
+
+async def test_unrelated_integrity_error_is_not_treated_as_duplicate_retry() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])
+ unrelated_error = IntegrityError(
+ "INSERT INTO workflow_runs",
+ None,
+ Exception('null value in column "organization_id" violates not-null constraint'),
+ )
+
+ result, _, mock_app = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ retried_by_results=[None, "wr_existing"],
+ run_workflow_error=unrelated_error,
+ )
+
+ assert result is None
+ assert mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by.await_count == 1
+
+
+async def test_fallback_retry_idempotency_returns_existing_retry_run() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ existing_retry_run_id="wr_existing",
+ )
+
+ assert result == "wr_existing"
+ run_workflow_mock.assert_not_awaited()
+
+
+ ("workflow_run", "block_scoped"),
+ [
+ (_workflow_run(parent_workflow_run_id="wr_parent"), False),
+ (_workflow_run(debug_session_id="debug_1"), False),
+ (_workflow_run(), True),
+ ],
+)
+async def test_fallback_retry_guards_skip_nested_debug_and_block_scoped_runs(
+ workflow_run: WorkflowRun,
+ block_scoped: bool,
+) -> None:
+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])
+
+ result, run_workflow_mock, _ = await _run_fallback_retry(
+ workflow_run=workflow_run,
+ parameters=[login],
+ block_scoped=block_scoped,
+ )
+
+ assert result is None
+ run_workflow_mock.assert_not_awaited()
+
+
+def test_credential_override_accepts_fallback_pool_and_rejects_non_pool_id() -> None:
+ service = WorkflowService()
+ workflow = _workflow([_credential_parameter(fallback_credential_ids=["cred_fb1"])])
+
+ overrides = service._get_run_credential_parameter_overrides(
+ workflow=workflow,
+ request_data={"login_cred": "cred_fb1"},
+ )
+
+ assert overrides == {"login_cred": "cred_fb1"}
+ with pytest.raises(SkyvernHTTPException, match="configured rotation or fallback credentials"):
+ service._get_run_credential_parameter_overrides(
+ workflow=workflow,
+ request_data={"login_cred": "cred_other"},
+ )
+
+
+async def test_service_resolves_fallback_only_credential_parameter_from_pinned_selection() -> None:
+ service = WorkflowService()
+ parameter = _credential_parameter(fallback_credential_ids=["cred_fb1"])
+
+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
+ mock_app.WORKFLOW_CONTEXT_MANAGER.workflow_run_contexts = {}
+ mock_app.DATABASE.workflow_run_credential_selections.get_selection = AsyncMock(return_value="cred_fb1")
+ selected = await service._resolve_credential_parameter_id(
+ parameter=parameter,
+ workflow_run_id="wr_retry",
+ organization_id="org_test",
+ workflow_permanent_id="wpid_test",
+ )
+
+ assert selected == "cred_fb1"
+
+
+async def test_context_resolves_fallback_only_credential_parameter_from_pinned_selection_once() -> None:
+ context = WorkflowRunContext(
+ workflow_title="Workflow",
+ workflow_id="wf_test",
+ workflow_permanent_id="wpid_test",
+ workflow_run_id="wr_retry",
+ aws_client=MagicMock(),
+ )
+ parameter = _credential_parameter(fallback_credential_ids=["cred_fb1"])
+
+ with patch("skyvern.forge.sdk.workflow.context_manager.app") as mock_app:
+ get_selection = AsyncMock(return_value="cred_fb1")
+ mock_app.DATABASE.workflow_run_credential_selections.get_selection = get_selection
+ first = await context.resolve_credential_parameter_id(parameter, "org_test")
+ second = await context.resolve_credential_parameter_id(parameter, "org_test")
+
+ assert first == "cred_fb1"
+ assert second == "cred_fb1"
+ get_selection.assert_awaited_once_with(workflow_run_id="wr_retry", parameter_key="login_cred")
+
+
+ ("pinned_selection", "expected"),
+ [
+ (None, "cred_runtime"),
+ ("cred_fb1", "cred_fb1"),
+ ],
+)
+async def test_context_fallback_only_parameter_applies_credential_id_indirection(
+ pinned_selection: str | None,
+ expected: str,
+) -> None:
+ context = WorkflowRunContext(
+ workflow_title="Workflow",
+ workflow_id="wf_test",
+ workflow_permanent_id="wpid_test",
+ workflow_run_id="wr_retry",
+ aws_client=MagicMock(),
+ )
+ context.parameters["credential_source"] = MagicMock()
+ context.values["credential_source"] = "cred_runtime"
+ parameter = _credential_parameter(
+ credential_id="credential_source",
+ fallback_credential_ids=["cred_fb1"],
+ )
+
+ with patch("skyvern.forge.sdk.workflow.context_manager.app") as mock_app:
+ mock_app.DATABASE.workflow_run_credential_selections.get_selection = AsyncMock(return_value=pinned_selection)
+ resolved = await context.resolve_credential_parameter_id(parameter, "org_test")
+
+ assert resolved == expected
+
+
+def test_yaml_to_credential_parameter_round_trip_preserves_fallback_fields() -> None:
+ yaml_definition = WorkflowDefinitionYAML(
+ parameters=[
+ CredentialParameterYAML(
+ key="login_cred",
+ credential_id="cred_primary",
+ fallback_credential_ids=["cred_fb1", "cred_fb2"],
+ fallback_trigger=ANY_FAILURE,
+ )
+ ],
+ blocks=[],
+ )
+
+ definition = convert_workflow_definition(yaml_definition, workflow_id="wf_test")
+ parameter = definition.parameters[0]
+
+ assert isinstance(parameter, CredentialParameter)
+ assert parameter.fallback_credential_ids == ["cred_fb1", "cred_fb2"]
+ assert parameter.fallback_trigger == ANY_FAILURE
+
+
+async def test_mark_workflow_run_as_failed_invokes_fallback_hook() -> None:
+ service = WorkflowService()
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+ service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
+
+ result = await service.mark_workflow_run_as_failed(
+ workflow_run_id="wr_failed",
+ failure_reason="login failed",
+ failure_category=[{"category": "AUTH_FAILURE"}],
+ )
+
+ assert result == workflow_run
+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)
+
+
+async def test_mark_workflow_run_as_terminated_invokes_fallback_hook() -> None:
+ service = WorkflowService()
+ workflow_run = _workflow_run(
+ status=WorkflowRunStatus.terminated,
+ failure_category=[{"category": "AUTH_FAILURE"}],
+ )
+ service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]
+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]
+
+ result = await service.mark_workflow_run_as_terminated(
+ workflow_run_id="wr_failed",
+ failure_reason="login failed",
+ failure_category=[{"category": "AUTH_FAILURE"}],
+ )
+
+ assert result == workflow_run
+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)
+
+
+async def test_fallback_hook_exception_does_not_raise() -> None:
+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])
+
+ with patch(
+ "skyvern.forge.sdk.workflow.service.maybe_start_credential_fallback_retry",
+ AsyncMock(side_effect=RuntimeError("boom")),
+ ):
+ await WorkflowService._start_credential_fallback_retry_best_effort(workflow_run)
diff --git a/tests/unit/test_credential_rotation.py b/tests/unit/test_credential_rotation.py
index 7261cf0c8..4a03af6f5 100644
--- a/tests/unit/test_credential_rotation.py
+++ b/tests/unit/test_credential_rotation.py
@@ -232,7 +232,7 @@ async def test_run_credential_override_rejects_credentials_outside_rotation_pool
service = WorkflowService()
workflow = _setup_workflow_with_rotating_credential()
- with pytest.raises(SkyvernHTTPException, match="configured rotation credentials"):
+ with pytest.raises(SkyvernHTTPException, match="configured rotation or fallback credentials"):
await service._apply_run_credential_parameter_overrides(
workflow=workflow,
workflow_run=_setup_workflow_run(),
diff --git a/tests/unit/test_runs_v2_route.py b/tests/unit/test_runs_v2_route.py
index ecfe16943..70be4eb7e 100644
--- a/tests/unit/test_runs_v2_route.py
+++ b/tests/unit/test_runs_v2_route.py
@@ -167,6 +167,7 @@ async def test_retry_workflow_run_replays_original_run_parameters(monkeypatch: p
browser_session_id="pbs_123",
browser_profile_id="bprof_123",
max_screenshot_scrolls=3,
+ max_elapsed_time_minutes=None,
extra_http_headers={"X-Test": "1"},
cdp_connect_headers={"X-CDP-Auth": "secret"},
browser_address="http://127.0.0.1:9222",
@@ -357,6 +358,7 @@ async def test_retry_workflow_run_replays_template_runs_as_templates(monkeypatch
browser_session_id=None,
browser_profile_id=None,
max_screenshot_scrolls=None,
+ max_elapsed_time_minutes=None,
extra_http_headers=None,
cdp_connect_headers=None,
browser_address=None,
diff --git a/tests/unit/test_saved_session_proxy_pin.py b/tests/unit/test_saved_session_proxy_pin.py
index a29de3b83..bb942eb0d 100644
--- a/tests/unit/test_saved_session_proxy_pin.py
+++ b/tests/unit/test_saved_session_proxy_pin.py
@@ -781,6 +781,8 @@ async def test_create_workflow_run_non_force_path_single_create_no_update(monkey
workflow_run_id=None,
trigger_type=None,
workflow_schedule_id=None,
+ retried_from_workflow_run_id=None,
+ fallback_attempt=None,
ignore_inherited_workflow_system_prompt=False,
copilot_session_id=None,
)
diff --git a/tests/unit/test_workflow_run_detail_template_cross_org.py b/tests/unit/test_workflow_run_detail_template_cross_org.py
index 900c3aa24..640b5a669 100644
--- a/tests/unit/test_workflow_run_detail_template_cross_org.py
+++ b/tests/unit/test_workflow_run_detail_template_cross_org.py
@@ -44,6 +44,7 @@ def _make_workflow_run() -> SimpleNamespace:
status=WorkflowRunStatus.running,
failure_reason=None,
failure_category=None,
+ retried_from_workflow_run_id=None,
proxy_location=None,
webhook_callback_url=None,
webhook_failure_reason=None,
@@ -90,6 +91,7 @@ async def test_template_run_detail_resolves_via_run_join(monkeypatch: pytest.Mon
workflow_runs=SimpleNamespace(
get_workflow_run_parameters=AsyncMock(return_value=[]),
get_workflow_run_block_errors=AsyncMock(return_value=[]),
+ get_workflow_run_retried_by=AsyncMock(return_value=None),
),
),
)