Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-12153): fallback credentials with automatic run retry (#7521)
Aaron Perez27c90a1df433
diff --git a/alembic/versions/2026_07_17_1752-d9ab2c4d41e5_add_credential_fallback_fields.py b/alembic/versions/2026_07_17_1752-d9ab2c4d41e5_add_credential_fallback_fields.pynew file mode 100644index 000000000..0a129e307--- /dev/null+++ b/alembic/versions/2026_07_17_1752-d9ab2c4d41e5_add_credential_fallback_fields.py@@ -0,0 +1,56 @@+"""add credential fallback fields++Revision ID: d9ab2c4d41e5+Revises: 1915b0e1126e+Create Date: 2026-07-17T17:52:48.243761+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "d9ab2c4d41e5"+down_revision: Union[str, None] = "1915b0e1126e"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ op.execute("SET LOCAL lock_timeout = '5s'")+ op.add_column("credential_parameters", sa.Column("fallback_credential_ids", sa.JSON(), nullable=True))+ op.add_column("credential_parameters", sa.Column("fallback_trigger", sa.String(), nullable=True))+ op.add_column("workflow_runs", sa.Column("retried_from_workflow_run_id", sa.String(), nullable=True))+ op.add_column("workflow_runs", sa.Column("fallback_attempt", sa.Integer(), nullable=True))++ with op.get_context().autocommit_block():+ op.execute("SET lock_timeout = '5s'")+ op.execute("SET statement_timeout = '1h'")+ op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_workflow_runs_retried_from_workflow_run_id")+ op.execute(+ """+ CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS ix_workflow_runs_retried_from_workflow_run_id+ ON workflow_runs (retried_from_workflow_run_id)+ WHERE retried_from_workflow_run_id IS NOT NULL+ """+ )+ op.execute("RESET statement_timeout")+ op.execute("RESET lock_timeout")+++def downgrade() -> None:+ with op.get_context().autocommit_block():+ op.execute("SET lock_timeout = '5s'")+ op.execute("SET statement_timeout = '1h'")+ op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_workflow_runs_retried_from_workflow_run_id")+ op.execute("RESET statement_timeout")+ op.execute("RESET lock_timeout")++ op.execute("SET LOCAL lock_timeout = '5s'")+ op.drop_column("workflow_runs", "fallback_attempt")+ op.drop_column("workflow_runs", "retried_from_workflow_run_id")+ op.drop_column("credential_parameters", "fallback_trigger")+ op.drop_column("credential_parameters", "fallback_credential_ids")diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex 49aee7a15..b7e6ed3ce 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -744,6 +744,7 @@ export type WorkflowRunApiResponse = {workflow_permanent_id: string;workflow_run_id: string;workflow_title: string | null;+ retried_from_workflow_run_id?: string | null;};export const TaskRunType = {@@ -810,6 +811,8 @@ export type WorkflowRunStatusApiResponse = {waiting_for_verification_code?: boolean;verification_code_identifier?: string | null;verification_code_polling_started_at?: string | null;+ retried_from_workflow_run_id?: string | null;+ retried_by_workflow_run_id?: string | null;};export type WorkflowRunStatusApiResponseWithWorkflow = {@@ -849,6 +852,8 @@ export type WorkflowRunStatusApiResponseWithWorkflow = {waiting_for_verification_code?: boolean;verification_code_identifier?: string | null;verification_code_polling_started_at?: string | null;+ retried_from_workflow_run_id?: string | null;+ retried_by_workflow_run_id?: string | null;};export type TaskGenerationApiResponse = {diff --git a/skyvern-frontend/src/components/CredentialFallbackRetryBadge.tsx b/skyvern-frontend/src/components/CredentialFallbackRetryBadge.tsxnew file mode 100644index 000000000..a720ee03b--- /dev/null+++ b/skyvern-frontend/src/components/CredentialFallbackRetryBadge.tsx@@ -0,0 +1,31 @@+import { UpdateIcon } from "@radix-ui/react-icons";++import { Tip } from "@/components/Tip";++type Props = {+ retriedFromWorkflowRunId: string | null | undefined;+};++const retryConfig = {+ icon: <UpdateIcon className="size-3.5 text-blue-400" />,+ label: "Automatic retry with fallback credential",+};++function CredentialFallbackRetryBadge({ retriedFromWorkflowRunId }: Props) {+ if (!retriedFromWorkflowRunId) {+ return null;+ }++ return (+ <Tip content={retryConfig.label}>+ <span+ aria-label={retryConfig.label}+ className="inline-flex shrink-0 items-center"+ >+ {retryConfig.icon}+ </span>+ </Tip>+ );+}++export { CredentialFallbackRetryBadge };diff --git a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsxindex 9dfc2396a..cd8a321d8 100644--- a/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx+++ b/skyvern-frontend/src/routes/workflows/RunWorkflowForm.tsx@@ -15,6 +15,7 @@ import { ProxyLocation } from "@/api/types";import { ProxySelector } from "@/components/ProxySelector";import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";import { Button } from "@/components/ui/button";+import { Badge } from "@/components/ui/badge";import {Accordion,AccordionContent,@@ -59,6 +60,7 @@ import { parseHeaderJson } from "@/util/secretHeaders";import { MAX_SCREENSHOT_SCROLLS_DEFAULT } from "./editor/nodes/Taskv2Node/types";import { getLabelForWorkflowParameterType } from "./editor/workflowEditorUtils";import {+ CredentialFallbackTrigger,CredentialParameter,WorkflowApiResponse,WorkflowBlock,@@ -74,7 +76,10 @@ import {parseJsonWorkflowParameterValue,validateJsonWorkflowParameterValue,} from "./utils";-import { getLoginCredentialInputs } from "./runWorkflowCredentials";+import {+ getLoginCredentialInputs,+ getRotatingCredentialIds,+} from "./runWorkflowCredentials";import { useCredentialsQuery } from "./hooks/useCredentialsQuery";import { visitWorkflowBlocks } from "./workflowBlockUtils";@@ -354,6 +359,47 @@ function getLoginCredentialDisplayText(};}+function FallbackCredentialList({+ fallbackCredentialIds,+ fallbackTrigger,+ credentialNamesById,+}: {+ fallbackCredentialIds: Array<string>;+ fallbackTrigger: CredentialFallbackTrigger | null;+ credentialNamesById: Map<string, string>;+}) {+ if (fallbackCredentialIds.length === 0) {+ return null;+ }++ const triggerText =+ fallbackTrigger === "any_failure"+ ? "for any reason"+ : "on credential failures";++ return (+ <div className="space-y-2">+ <ol className="space-y-1">+ {fallbackCredentialIds.map((credentialId, index) => (+ <li+ key={credentialId}+ className="flex min-w-0 items-center gap-2 rounded border border-slate-700/60 bg-slate-950/40 px-2 py-1.5 text-xs text-slate-200"+ >+ <span className="shrink-0 text-slate-400">{index + 1}.</span>+ <span className="truncate">+ {credentialNamesById.get(credentialId) ?? credentialId}+ </span>+ </li>+ ))}+ </ol>+ <p className="text-xs text-slate-400">+ If the run fails {triggerText}, Skyvern retries it automatically with+ the next fallback.+ </p>+ </div>+ );+}+type RunWorkflowFormType = Record<string, unknown> & {webhookCallbackUrl: string;proxyLocation: ProxyLocation;@@ -764,11 +810,18 @@ function RunWorkflowForm({<header><h1 className="text-lg">Login credentials</h1></header>- {loginCredentialInputs.map(({ parameter, loginBlockLabels }) => {+ {loginCredentialInputs.map((input) => {+ const {+ parameter,+ loginBlockLabels,+ fallbackCredentialIds,+ fallbackTrigger,+ } = input;const { title, description } = getLoginCredentialDisplayText(parameter,loginBlockLabels,);+ const hasFallbacks = fallbackCredentialIds.length > 0;if (parameter.parameter_type === WorkflowParameterTypes.Workflow@@ -829,6 +882,47 @@ function RunWorkflowForm({);}+ const credentialIds = getRotatingCredentialIds(parameter);++ if (credentialIds.length <= 1) {+ const primaryCredentialId =+ credentialIds[0] ?? parameter.credential_id;+ return (+ <div key={parameter.key} className="flex gap-16">+ <div className="w-72 shrink-0 text-slate-50">+ <div className="flex items-center gap-2 text-lg">+ {title}+ <span className="text-sm text-slate-400">+ credential+ </span>+ </div>+ <h2 className="text-sm text-slate-400">{description}</h2>+ </div>+ <div className="w-full space-y-2">+ <div className="flex min-w-0 items-center gap-2 rounded border border-slate-700/60 bg-slate-950/40 px-2 py-1.5 text-xs text-slate-200">+ <span className="min-w-0 truncate">+ {credentialNamesById.get(primaryCredentialId) ??+ primaryCredentialId}+ </span>+ {hasFallbacks && (+ <Badge+ variant="outline"+ className="shrink-0 text-xs font-normal"+ >+ Primary+ </Badge>+ )}+ </div>+ <FallbackCredentialList+ fallbackCredentialIds={fallbackCredentialIds}+ fallbackTrigger={fallbackTrigger}+ credentialNamesById={credentialNamesById}+ />+ </div>+ </div>+ );+ }+return (<FormFieldkey={parameter.key}@@ -845,6 +939,14 @@ function RunWorkflowForm({credentialNamesById={credentialNamesById}title={title}description={description}+ showPrimaryBadge={hasFallbacks}+ fallbackContent={+ <FallbackCredentialList+ fallbackCredentialIds={fallbackCredentialIds}+ fallbackTrigger={fallbackTrigger}+ credentialNamesById={credentialNamesById}+ />+ }/>)}/>diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsxindex d24c2d25a..23be1e093 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.test.tsx@@ -1,14 +1,16 @@// @vitest-environment jsdom-import { cleanup, render, screen } from "@testing-library/react";+import { cleanup, render, screen, within } from "@testing-library/react";import { afterEach, describe, expect, it, vi } from "vitest";import { MemoryRouter, Route, Routes, useLocation } from "react-router-dom";+import { Status, type WorkflowRunApiResponse } from "@/api/types";import CloudContext from "@/store/CloudContext";import { PageSlotsProvider, type PageSlots } from "@/store/PageSlots";import { WorkflowPage } from "./WorkflowPage";-const { mockFeatureFlagEnabled } = vi.hoisted(() => ({+const { mockFeatureFlagEnabled, mockWorkflowRunsQuery } = vi.hoisted(() => ({mockFeatureFlagEnabled: vi.fn(),+ mockWorkflowRunsQuery: vi.fn(),}));vi.mock("posthog-js/react", () => ({@@ -63,10 +65,7 @@ vi.mock("./hooks/useWorkflowQuery", () => ({}));vi.mock("./hooks/useWorkflowRunsQuery", () => ({- useWorkflowRunsQuery: () => ({- data: [],- isLoading: false,- }),+ useWorkflowRunsQuery: () => mockWorkflowRunsQuery(),}));vi.mock("./hooks/useWorkflowTagsBatchQuery", () => ({@@ -131,6 +130,7 @@ type RenderOptions = {analyticsFlagEnabled?: boolean;pageSlots?: PageSlots;initialEntries?: Array<string>;+ workflowRuns?: Array<WorkflowRunApiResponse>;};function LocationProbe() {@@ -143,8 +143,13 @@ function renderWorkflowPage({analyticsFlagEnabled = true,pageSlots = {},initialEntries = ["/workflows/wpid_abc123"],+ workflowRuns = [],}: RenderOptions = {}) {mockFeatureFlagEnabled.mockReturnValue(analyticsFlagEnabled);+ mockWorkflowRunsQuery.mockReturnValue({+ data: workflowRuns,+ isLoading: false,+ });return render(<CloudContext.Provider value={isCloud}>@@ -167,6 +172,28 @@ function renderWorkflowPage({);}+function makeWorkflowRun(+ overrides: Partial<WorkflowRunApiResponse>,+): WorkflowRunApiResponse {+ return {+ created_at: "2026-06-01T12:00:00.000Z",+ failure_reason: null,+ started_at: "2026-06-01T12:00:01.000Z",+ finished_at: "2026-06-01T12:00:10.000Z",+ modified_at: "2026-06-01T12:00:10.000Z",+ proxy_location: null,+ script_run: false,+ status: Status.Completed,+ webhook_callback_url: "",+ workflow_id: "wf_abc123",+ workflow_permanent_id: "wpid_abc123",+ workflow_run_id: "wr_default",+ workflow_title: "Test Workflow",+ retried_from_workflow_run_id: null,+ ...overrides,+ };+}+describe("WorkflowPage analytics button", () => {it("shows the Analytics link for cloud users when the dashboard flag is enabled", () => {renderWorkflowPage();@@ -230,4 +257,32 @@ describe("WorkflowPage analytics button", () => {expect(search).toContain("from=2026-06-01");expect(search).toContain("to=2026-06-03");});++ it("shows the fallback retry badge only for runs retried from another run", () => {+ renderWorkflowPage({+ workflowRuns: [+ makeWorkflowRun({+ workflow_run_id: "wr_retry",+ retried_from_workflow_run_id: "wr_original",+ }),+ makeWorkflowRun({ workflow_run_id: "wr_original" }),+ ],+ });++ const retryRow = screen.getByText("wr_retry").closest("tr");+ const originalRow = screen.getByText("wr_original").closest("tr");++ expect(retryRow).not.toBeNull();+ expect(originalRow).not.toBeNull();+ expect(+ within(retryRow!).getByLabelText(+ "Automatic retry with fallback credential",+ ),+ ).not.toBeNull();+ expect(+ within(originalRow!).queryByLabelText(+ "Automatic retry with fallback credential",+ ),+ ).toBeNull();+ });});diff --git a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsxindex 8f1dffeb1..4dfe8317d 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowPage.tsx@@ -10,6 +10,7 @@ import {import { Tip } from "@/components/Tip";import { Status, WorkflowRunStatusApiResponse } from "@/api/types";import { StatusBadge } from "@/components/StatusBadge";+import { CredentialFallbackRetryBadge } from "@/components/CredentialFallbackRetryBadge";import { StatusFilterDropdown } from "@/components/StatusFilterDropdown";import { Button } from "@/components/ui/button";import {@@ -490,7 +491,14 @@ function WorkflowPage() {</div></TableCell><TableCell>- <StatusBadge status={workflowRun.status} />+ <div className="flex items-center gap-2">+ <StatusBadge status={workflowRun.status} />+ <CredentialFallbackRetryBadge+ retriedFromWorkflowRunId={+ workflowRun.retried_from_workflow_run_id+ }+ />+ </div></TableCell><TableCellclassName="text-muted-foreground"diff --git a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsxindex c72cda03e..78ba90efb 100644--- a/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx+++ b/skyvern-frontend/src/routes/workflows/WorkflowRun.tsx@@ -4,6 +4,7 @@ import { getClient } from "@/api/AxiosClient";import { ProxyLocation, Status } from "@/api/types";import { FailureCategoryBadge } from "@/components/FailureCategoryBadge";import { StatusBadge } from "@/components/StatusBadge";+import { CredentialFallbackRetryBadge } from "@/components/CredentialFallbackRetryBadge";import {SwitchBarNavigation,type SwitchBarNavigationOption,@@ -312,6 +313,8 @@ function WorkflowRun() {workflowRun?.status === Status.Terminated? "Termination Reason": "Failure Reason";+ const siblingRunLinkClassName =+ "font-mono text-sm text-neutral-600 hover:text-neutral-950 hover:underline hover:underline-offset-2 dark:text-slate-400 dark:hover:text-slate-200";const finallyBlockInTimeline = finallyBlockLabel? workflowRunTimeline?.find(@@ -563,10 +566,14 @@ function WorkflowRun() {{workflowRunIsLoading ? (<Skeleton className="h-8 w-28" />) : workflowRun ? (- <StatusBadge- className="mt-[0.27rem]"- status={workflowRun?.status}- />+ <div className="mt-[0.27rem] flex items-center gap-2">+ <StatusBadge status={workflowRun?.status} />+ <CredentialFallbackRetryBadge+ retriedFromWorkflowRunId={+ workflowRun.retried_from_workflow_run_id+ }+ />+ </div>) : null}</div><h2 className="text-2xl text-neutral-600 dark:text-slate-400">@@ -575,6 +582,26 @@ function WorkflowRun() {{workflowRunId ? (<RunTagsEditor workflowRunId={workflowRunId} />) : null}+ {workflowRun && workflowPermanentId && (+ <div className="flex flex-wrap gap-x-4 gap-y-1">+ {workflowRun.retried_from_workflow_run_id && (+ <Link+ className={siblingRunLinkClassName}+ to={`/agents/${workflowPermanentId}/${workflowRun.retried_from_workflow_run_id}/overview`}+ >+ Retried from {workflowRun.retried_from_workflow_run_id}+ </Link>+ )}+ {workflowRun.retried_by_workflow_run_id && (+ <Link+ className={siblingRunLinkClassName}+ to={`/agents/${workflowPermanentId}/${workflowRun.retried_by_workflow_run_id}/overview`}+ >+ Retried by {workflowRun.retried_by_workflow_run_id}+ </Link>+ )}+ </div>+ )}{workflowRun &&(workflowRun.started_at ||workflowRun.finished_at ||diff --git a/skyvern-frontend/src/routes/workflows/components/RotatingCredentialField.tsx b/skyvern-frontend/src/routes/workflows/components/RotatingCredentialField.tsxindex 1b229cd2c..d2df8996c 100644--- a/skyvern-frontend/src/routes/workflows/components/RotatingCredentialField.tsx+++ b/skyvern-frontend/src/routes/workflows/components/RotatingCredentialField.tsx@@ -1,3 +1,5 @@+import { type ReactNode } from "react";+import { Badge } from "@/components/ui/badge";import { FormControl, FormItem, FormLabel } from "@/components/ui/form";import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group";@@ -19,6 +21,8 @@ type RotatingCredentialFieldProps = {credentialNamesById: Map<string, string>;title: string;description: string;+ showPrimaryBadge?: boolean;+ fallbackContent?: ReactNode;};function formatCredentialSelectionStrategy(@@ -38,6 +42,8 @@ function RotatingCredentialField({credentialNamesById,title,description,+ showPrimaryBadge = false,+ fallbackContent,}: RotatingCredentialFieldProps) {const credentialIds = getRotatingCredentialIds(parameter);const forcedCredentialId = typeof value === "string" ? value : "";@@ -127,27 +133,38 @@ function RotatingCredentialField({</Badge>)}</div>- <Select- disabled={mode !== "force"}- value={forceSelectValue}- onValueChange={(value) => onChange(value)}- >- <SelectTrigger className="w-full">- <SelectValue placeholder="Select a credential" />- </SelectTrigger>- <SelectContent>- {credentialIds.map((credentialId) => (- <SelectItem key={credentialId} value={credentialId}>- {credentialNamesById.get(credentialId) ??- credentialId}- </SelectItem>- ))}- </SelectContent>- </Select>+ <div className="flex items-center gap-2">+ <Select+ disabled={mode !== "force"}+ value={forceSelectValue}+ onValueChange={(value) => onChange(value)}+ >+ <SelectTrigger className="w-full">+ <SelectValue placeholder="Select a credential" />+ </SelectTrigger>+ <SelectContent>+ {credentialIds.map((credentialId) => (+ <SelectItem key={credentialId} value={credentialId}>+ {credentialNamesById.get(credentialId) ??+ credentialId}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>+ {showPrimaryBadge && (+ <Badge+ variant="outline"+ className="shrink-0 text-xs font-normal"+ >+ Primary+ </Badge>+ )}+ </div></div></label></RadioGroup></FormControl>+ {fallbackContent}</div></div></FormItem>diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.test.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.test.tsxindex 94cd05ebf..054e04eab 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.test.tsx@@ -244,3 +244,140 @@ describe("LoginBlockCredentialSelector", () => {);});});++describe("LoginBlockCredentialSelector fallback picker", () => {+ beforeEach(() => {+ mocks.useCredentialsQuery.mockReturnValue({+ data: [+ credential("cred_primary", "primary_credential"),+ credential("cred_backup_1", "backup_credential_1"),+ ],+ isFetching: false,+ isLoading: false,+ });+ useWorkflowParametersStore.setState({+ parameters: [+ {+ key: "portal_credential",+ parameterType: "credential",+ credentialId: "cred_primary",+ },+ ],+ });+ });++ function renderSelector() {+ return renderInCloud(+ <LoginBlockCredentialSelector+ nodeId="node-1"+ value="portal_credential"+ />,+ );+ }++ async function openFallbackPicker() {+ renderSelector();+ fireEvent.click(screen.getByText("Add fallback credentials"));+ const trigger = await screen.findByRole("button", {+ name: /add fallback credentials/i,+ });+ fireEvent.click(trigger);+ await screen.findByPlaceholderText("Search credentials...");+ }++ it("hides fallback controls when credential rotation is configured", () => {+ useWorkflowParametersStore.setState({+ parameters: [+ {+ key: "portal_credential",+ parameterType: "credential",+ credentialId: "cred_primary",+ credentialIds: ["cred_primary", "cred_backup_1"],+ },+ ],+ });++ renderSelector();++ expect(screen.queryByText("Fallback credentials")).toBeNull();+ expect(screen.queryByText("Add fallback credentials")).toBeNull();+ expect(+ screen.getByText(+ "Fallback credentials can't be combined with credential rotation.",+ ),+ ).toBeTruthy();+ });++ it("hides credential rotation when fallback credentials are configured", () => {+ useWorkflowParametersStore.setState({+ parameters: [+ {+ key: "portal_credential",+ parameterType: "credential",+ credentialId: "cred_primary",+ fallbackCredentialIds: ["cred_backup_1"],+ },+ ],+ });++ renderSelector();++ expect(+ screen.queryByText("Rotate between multiple credentials"),+ ).toBeNull();+ expect(+ screen.getByText(+ "Credential rotation can't be combined with fallback credentials.",+ ),+ ).toBeTruthy();+ });++ it("Escape closes only the picker and does not reach window listeners", async () => {+ await openFallbackPicker();++ const windowEscapeSpy = vi.fn();+ const listener = (event: KeyboardEvent) => {+ if (event.key === "Escape") {+ windowEscapeSpy();+ }+ };+ window.addEventListener("keydown", listener);+ try {+ fireEvent.keyDown(screen.getByPlaceholderText("Search credentials..."), {+ key: "Escape",+ });++ await waitFor(() => {+ expect(+ screen.queryByPlaceholderText("Search credentials..."),+ ).toBeNull();+ });+ expect(windowEscapeSpy).not.toHaveBeenCalled();+ } finally {+ window.removeEventListener("keydown", listener);+ }+ });++ it("selecting a fallback option writes it to the parameter store", async () => {+ await openFallbackPicker();++ fireEvent.click(await screen.findByText("backup_credential_1"));++ await waitFor(() => {+ const parameter = useWorkflowParametersStore+ .getState()+ .parameters.find((p) => p.key === "portal_credential");+ expect(parameter).toMatchObject({+ fallbackCredentialIds: ["cred_backup_1"],+ });+ });+ });++ it("the picker popover is modal so outside clicks cannot hit the canvas", async () => {+ await openFallbackPicker();++ await waitFor(() => {+ expect(document.body.style.pointerEvents).toBe("none");+ });+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsxindex 3c41832fd..e2de6b242 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx@@ -12,6 +12,13 @@ import {PopoverContent,PopoverTrigger,} from "@/components/ui/popover";+import {+ Select,+ SelectContent,+ SelectItem,+ SelectTrigger,+ SelectValue,+} from "@/components/ui/select";import { Skeleton } from "@/components/ui/skeleton";import { Switch } from "@/components/ui/switch";import { useCredentialsQuery } from "@/routes/workflows/hooks/useCredentialsQuery";@@ -26,10 +33,12 @@ import {ChevronDownIcon,Cross2Icon,ExclamationTriangleIcon,+ PlusIcon,ReloadIcon,} from "@radix-ui/react-icons";import type { CredentialApiResponse } from "@/api/types";import { getHostname } from "@/util/getHostname";+import { type CredentialFallbackTrigger } from "@/routes/workflows/types/workflowTypes";import {CredentialModalTypes,useCredentialModalState,@@ -50,6 +59,7 @@ import {parameterIsAzureVaultCredential,isAutoGeneratedCredentialKey,computeWrappedCredentialIds,+ type SkyvernCredential,} from "../../types";import { cn } from "@/util/utils";import { keepPreviousData } from "@tanstack/react-query";@@ -65,6 +75,7 @@ import {} from "@/routes/workflows/components/CredentialCombobox";const ROTATION_SELECTION_STRATEGY = "round_robin" as const;+const DEFAULT_FALLBACK_TRIGGER = "credential_failures" as const;type Props = {nodeId: string;@@ -80,7 +91,7 @@ type Props = {currentInstructions?: string;};-type RotationCredentialRowProps = {+type CredentialListRowProps = {credentialId: string;credential: CredentialApiResponse | undefined;index: number;@@ -89,14 +100,14 @@ type RotationCredentialRowProps = {onRemove: () => void;};-function RotationCredentialRow({+function CredentialListRow({credentialId,credential,index,editable,canRemove,onRemove,-}: RotationCredentialRowProps) {+}: CredentialListRowProps) {const detailQuery = useCredentialQuery(credentialId, {enabled: !credential,});@@ -172,6 +183,7 @@ function LoginBlockCredentialSelector({const [rotationDraft, setRotationDraft] = useState(false);const [rotationQuery, setRotationQuery] = useState("");const [debouncedRotationQuery] = useDebounce(rotationQuery, 300);+ const [fallbackDraft, setFallbackDraft] = useState(false);const {parameters: workflowParameters,setParameters: setWorkflowParameters,@@ -203,6 +215,7 @@ function LoginBlockCredentialSelector({useEffect(() => {setRotationDraft(false);+ setFallbackDraft(false);}, [value]);// Determine which credential is currently selected (by credential_id)@@ -282,6 +295,14 @@ function LoginBlockCredentialSelector({});}, [selectedSkyvernCredentialParameter, selectedCredentialId]);+ const fallbackCredentialIds = useMemo(() => {+ return (+ selectedSkyvernCredentialParameter?.fallbackCredentialIds ?? []+ ).filter((id, index, ids): id is string => {+ return typeof id === "string" && ids.indexOf(id) === index;+ });+ }, [selectedSkyvernCredentialParameter]);+const rotationIsSaved = rotationCredentialIds.length > 1;const rotationMode =Boolean(selectedSkyvernCredentialParameter) &&@@ -295,6 +316,10 @@ function LoginBlockCredentialSelector({search: debouncedRotationQuery.trim() || undefined,placeholderData: keepPreviousData,});+ const fallbackIsSaved = fallbackCredentialIds.length > 0;+ const fallbackMode =+ Boolean(selectedSkyvernCredentialParameter) &&+ (fallbackDraft || fallbackIsSaved);const workflowStartNode = useMemo<{ id: string; data: WorkflowStartNodeData } | undefined@@ -344,6 +369,19 @@ function LoginBlockCredentialSelector({browserProfileUrl: credential.tested_url ?? null,}));+ const fallbackCredentialOptions = credentials+ .filter(+ (credential) =>+ !fallbackCredentialIds.includes(credential.credential_id) &&+ credential.credential_id !== selectedCredentialId,+ )+ .map((credential) => ({+ label: credential.name,+ value: credential.credential_id,+ hasBrowserProfile: !!credential.browser_profile_id,+ browserProfileUrl: credential.tested_url ?? null,+ }));+// Auto-generated Skyvern wrappers are hidden (they back a direct credential// pick); user-authored Skyvern params and external vault params are listed.// Workflow credential_id params are authored in the Parameters panel, not here.@@ -419,6 +457,70 @@ function LoginBlockCredentialSelector({setHasChanges(true);};+ const writeFallbackCredentialIds = (credentialIds: Array<string>) => {+ if (!selectedSkyvernCredentialParameter) {+ return;+ }++ const orderedCredentialIds = credentialIds.filter((id, index, ids) => {+ return ids.indexOf(id) === index;+ });++ const newParameters = workflowParameters.map((parameter) => {+ if (+ parameter.key !== selectedSkyvernCredentialParameter.key ||+ parameter.parameterType !== "credential" ||+ !parameterIsSkyvernCredential(parameter)+ ) {+ return parameter;+ }++ if (orderedCredentialIds.length === 0) {+ return {+ ...parameter,+ fallbackCredentialIds: null,+ fallbackTrigger: null,+ };+ }++ return {+ ...parameter,+ fallbackCredentialIds: orderedCredentialIds,+ fallbackTrigger: parameter.fallbackTrigger ?? null,+ };+ });++ setWorkflowParameters(newParameters);+ setHasChanges(true);+ };++ const writeFallbackTrigger = (fallbackTrigger: CredentialFallbackTrigger) => {+ if (+ !selectedSkyvernCredentialParameter ||+ fallbackCredentialIds.length === 0+ ) {+ return;+ }++ const newParameters = workflowParameters.map((parameter) => {+ if (+ parameter.key !== selectedSkyvernCredentialParameter.key ||+ parameter.parameterType !== "credential" ||+ !parameterIsSkyvernCredential(parameter)+ ) {+ return parameter;+ }++ return {+ ...parameter,+ fallbackTrigger,+ };+ });++ setWorkflowParameters(newParameters);+ setHasChanges(true);+ };+const toggleRotationCredential = (credentialId: string) => {if (!editable) return;const nextCredentialIds = rotationCredentialIds.includes(credentialId)@@ -435,6 +537,22 @@ function LoginBlockCredentialSelector({}};+ const addFallbackCredential = (credentialId: string) => {+ if (!editable || fallbackCredentialIds.includes(credentialId)) return;+ writeFallbackCredentialIds([...fallbackCredentialIds, credentialId]);+ };++ const removeFallbackCredential = (credentialId: string) => {+ if (!editable) return;+ const nextCredentialIds = fallbackCredentialIds.filter(+ (id) => id !== credentialId,+ );+ writeFallbackCredentialIds(nextCredentialIds);+ if (nextCredentialIds.length === 0) {+ setFallbackDraft(false);+ }+ };+const disableRotation = () => {if (!editable) return;if (rotationCredentialIds.length > 0) {@@ -464,17 +582,30 @@ function LoginBlockCredentialSelector({const loginNodes = nodes.filter((node) => node.id !== nodeId).filter(isLoginNode);- const currentParameter = workflowParameters.find((parameter) => {- if (parameter.parameterType !== "credential") return false;- if (!parameterIsSkyvernCredential(parameter)) return false;- if (!isAutoGeneratedCredentialKey(parameter.key)) return false;- return parameter.key === value;- });+ const currentParameter = workflowParameters.find(+ (parameter): parameter is SkyvernCredential => {+ if (parameter.parameterType !== "credential") return false;+ if (!parameterIsSkyvernCredential(parameter)) return false;+ if (!isAutoGeneratedCredentialKey(parameter.key)) return false;+ return parameter.key === value;+ },+ );const isUsedInOtherLoginNodes =value &&loginNodes.some((node) => node.data.parameterKeys.includes(value));+ const deleteOldParameter = currentParameter && !isUsedInOtherLoginNodes;- if (currentParameter && !isUsedInOtherLoginNodes) {+ // The new primary may not be its own fallback.+ const carriedFallbackCredentialIds = deleteOldParameter+ ? (currentParameter.fallbackCredentialIds ?? []).filter(+ (credentialId) => credentialId !== newValue,+ )+ : [];+ const carriedFallbackTrigger = deleteOldParameter+ ? (currentParameter.fallbackTrigger ?? null)+ : null;++ if (deleteOldParameter) {newParameters = newParameters.filter((parameter) => parameter.key !== value,);@@ -495,7 +626,30 @@ function LoginBlockCredentialSelector({});if (existingParameter) {+ // Reuse the existing parameter; carry the outgoing wrapper's fallback+ // config onto it when it has none of its own.parameterKeyToUse = existingParameter.key;+ if (+ carriedFallbackCredentialIds.length > 0 &&+ existingParameter.parameterType === "credential" &&+ parameterIsSkyvernCredential(existingParameter) &&+ (existingParameter.fallbackCredentialIds?.length ?? 0) === 0+ ) {+ newParameters = newParameters.map((parameter) => {+ if (+ parameter.key !== existingParameter.key ||+ parameter.parameterType !== "credential" ||+ !parameterIsSkyvernCredential(parameter)+ ) {+ return parameter;+ }+ return {+ ...parameter,+ fallbackCredentialIds: carriedFallbackCredentialIds,+ fallbackTrigger: carriedFallbackTrigger,+ };+ });+ }} else {const existingKeys = newParameters.map((parameter) => parameter.key);const newKey = generateDefaultCredentialParameterKey(existingKeys);@@ -506,6 +660,10 @@ function LoginBlockCredentialSelector({parameterType: "credential",credentialId: newValue,key: newKey,+ ...(carriedFallbackCredentialIds.length > 0 && {+ fallbackCredentialIds: carriedFallbackCredentialIds,+ fallbackTrigger: carriedFallbackTrigger,+ }),},];}@@ -561,11 +719,155 @@ function LoginBlockCredentialSelector({return undefined;};+ const fallbackTrigger =+ selectedSkyvernCredentialParameter?.fallbackTrigger ??+ DEFAULT_FALLBACK_TRIGGER;++ const fallbackSection = selectedSkyvernCredentialParameter ? (+ fallbackMode ? (+ <div className="mt-3 space-y-2 border-t border-border pt-3">+ <div className="space-y-1">+ <span className="text-xs text-foreground dark:text-slate-300">+ Fallback credentials+ </span>+ <p className="text-[11px] leading-4 text-muted-foreground dark:text-slate-500">+ If a run fails, Skyvern automatically retries it with the next+ fallback credential.+ </p>+ </div>+ {/* modal: an outside click must only dismiss the picker — non-modal+ popovers let the same click hit the canvas pane, which deselects+ the block and closes the whole config panel. */}+ <Popover modal>+ <PopoverTrigger asChild>+ <Button+ type="button"+ variant="outline"+ className="h-auto min-h-10 w-full justify-between px-3 py-2 text-left text-xs font-normal"+ disabled={!editable || fallbackCredentialOptions.length === 0}+ >+ <span className="truncate">+ {fallbackCredentialIds.length > 0+ ? `${fallbackCredentialIds.length} fallback ${+ fallbackCredentialIds.length === 1+ ? "credential"+ : "credentials"+ }`+ : "Add fallback credentials"}+ </span>+ <ChevronDownIcon className="ml-2 size-4 shrink-0 text-muted-foreground" />+ </Button>+ </PopoverTrigger>+ <PopoverContent+ className="w-[28rem] p-0"+ align="start"+ // Escape must only dismiss the picker; without this it bubbles to+ // the editor's window listener, which closes the whole block panel.+ onEscapeKeyDown={(event) => event.stopPropagation()}+ >+ <Command>+ <CommandInput placeholder="Search credentials..." />+ <CommandList>+ <CommandEmpty>No credentials found.</CommandEmpty>+ <CommandGroup>+ {fallbackCredentialOptions.map((option) => (+ <CommandItem+ key={option.value}+ className="cursor-pointer"+ onSelect={() => addFallbackCredential(option.value)}+ >+ <div+ className={cn(+ "mr-2 flex size-4 items-center justify-center rounded-sm border border-primary",+ "opacity-50 [&_svg]:invisible",+ )}+ >+ <CheckIcon className="size-4" />+ </div>+ <div className="flex min-w-0 items-center gap-2">+ <span className="truncate">{option.label}</span>+ {option.hasBrowserProfile ? (+ <span className="rounded bg-green-100 px-1.5 py-0.5 text-[10px] text-green-700 dark:bg-green-900/40 dark:text-green-400">+ saved-profile+ </span>+ ) : null}+ {option.browserProfileUrl ? (+ <span className="truncate text-[10px] text-muted-foreground">+ {getHostname(option.browserProfileUrl)}+ </span>+ ) : null}+ </div>+ </CommandItem>+ ))}+ </CommandGroup>+ </CommandList>+ </Command>+ </PopoverContent>+ </Popover>++ {fallbackCredentialIds.length > 0 ? (+ <>+ <div className="space-y-1">+ {fallbackCredentialIds.map((credentialId, index) => (+ <CredentialListRow+ key={credentialId}+ credentialId={credentialId}+ credential={credentials.find(+ (credential) => credential.credential_id === credentialId,+ )}+ index={index}+ editable={editable}+ canRemove+ onRemove={() => removeFallbackCredential(credentialId)}+ />+ ))}+ </div>+ <div className="space-y-1">+ <span className="text-xs text-foreground dark:text-slate-300">+ Retry failed runs+ </span>+ <Select+ disabled={!editable}+ value={fallbackTrigger}+ onValueChange={(value) =>+ writeFallbackTrigger(value as CredentialFallbackTrigger)+ }+ >+ <SelectTrigger className="h-9 w-full text-xs">+ <SelectValue />+ </SelectTrigger>+ <SelectContent>+ <SelectItem value="credential_failures">+ On credential failures+ </SelectItem>+ <SelectItem value="any_failure">On any failure</SelectItem>+ </SelectContent>+ </Select>+ </div>+ </>+ ) : null}+ </div>+ ) : (+ <button+ type="button"+ className="mt-2 inline-flex items-center gap-1 text-xs text-muted-foreground transition-colors hover:text-foreground disabled:pointer-events-none disabled:opacity-50 dark:hover:text-slate-200"+ disabled={!editable}+ onClick={() => setFallbackDraft(true)}+ >+ <PlusIcon className="size-3" />+ Add fallback credentials+ </button>+ )+ ) : null;+if (rotationMode) {return (<><div className="space-y-2">- <Popover>+ {/* modal: an outside click must only dismiss the picker — non-modal+ popovers let the same click hit the canvas pane, which deselects+ the block and closes the whole config panel. */}+ <Popover modal><PopoverTrigger asChild><Buttontype="button"@@ -581,7 +883,13 @@ function LoginBlockCredentialSelector({<ChevronDownIcon className="ml-2 size-4 shrink-0 text-muted-foreground" /></Button></PopoverTrigger>- <PopoverContent className="w-[28rem] p-0" align="start">+ <PopoverContent+ className="w-[28rem] p-0"+ align="start"+ // Escape must only dismiss the picker; without this it bubbles to+ // the editor's window listener, which closes the whole block panel.+ onEscapeKeyDown={(event) => event.stopPropagation()}+ ><Command shouldFilter={false}><CommandInputplaceholder="Search credentials..."@@ -643,7 +951,7 @@ function LoginBlockCredentialSelector({<div className="space-y-1">{rotationCredentialIds.map((credentialId, index) => (- <RotationCredentialRow+ <CredentialListRowkey={credentialId}credentialId={credentialId}credential={rotationCredentials.find(@@ -687,6 +995,10 @@ function LoginBlockCredentialSelector({>Use single credential</button>+ <p className="text-[11px] leading-4 text-muted-foreground dark:text-slate-500">+ Fallback credentials can't be combined with credential+ rotation.+ </p></div><CredentialsModalonCredentialCreated={(id) => {@@ -734,10 +1046,13 @@ function LoginBlockCredentialSelector({: "h-auto min-h-10 w-full",}}/>- {!rotationMode &&- selectedSkyvernCredentialParameter &&- selectedCredentialId &&- !isCredentialMissing ? (+ {fallbackMode ? (+ <p className="mt-2 text-[11px] leading-4 text-muted-foreground dark:text-slate-500">+ Credential rotation can't be combined with fallback credentials.+ </p>+ ) : selectedSkyvernCredentialParameter &&+ selectedCredentialId &&+ !isCredentialMissing ? (<buttontype="button"className="mt-2 inline-flex items-center gap-1 text-xs text-muted-foreground transition-colors hover:text-foreground disabled:pointer-events-none disabled:opacity-50 dark:hover:text-slate-200"@@ -748,6 +1063,7 @@ function LoginBlockCredentialSelector({Rotate between multiple credentials</button>) : null}+ {fallbackSection}<CredentialsModalonCredentialCreated={(id) => {if (!editable) return;diff --git a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsxindex 64856b608..27ddf8f36 100644--- a/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/panels/WorkflowParameterEditPanel.tsx@@ -33,6 +33,7 @@ import {parameterIsAzureVaultCredential,AUTO_GENERATED_CREDENTIAL_KEY_PATTERN,} from "../types";+import { applySkyvernCredentialEdit } from "../utils";import { getDefaultValueForParameterType } from "../workflowEditorUtils";import { validateBitwardenLoginCredential } from "./util";import { HelpTooltip } from "@/components/HelpTooltip";@@ -1173,12 +1174,13 @@ function WorkflowParameterEditPanel({});return;}- onSave({- key,- parameterType: "credential",- credentialId,- description,- });+ onSave(+ applySkyvernCredentialEdit(initialValues, {+ key,+ credentialId,+ description,+ }),+ );return;}diff --git a/skyvern-frontend/src/routes/workflows/editor/types.ts b/skyvern-frontend/src/routes/workflows/editor/types.tsindex 7b9864aa5..7053beea2 100644--- a/skyvern-frontend/src/routes/workflows/editor/types.ts+++ b/skyvern-frontend/src/routes/workflows/editor/types.ts@@ -1,4 +1,5 @@import {+ CredentialFallbackTrigger,CredentialSelectionStrategy,WorkflowParameterValueType,} from "../types/workflowTypes";@@ -25,6 +26,8 @@ export type SkyvernCredential = {credentialId: string;credentialIds?: Array<string> | null;selectionStrategy?: CredentialSelectionStrategy | null;+ fallbackCredentialIds?: Array<string> | null;+ fallbackTrigger?: CredentialFallbackTrigger | null;dataType?: typeof WorkflowParameterValueType.CredentialId;};diff --git a/skyvern-frontend/src/routes/workflows/editor/utils.test.ts b/skyvern-frontend/src/routes/workflows/editor/utils.test.tsindex 7efa34fe4..648b3da70 100644--- a/skyvern-frontend/src/routes/workflows/editor/utils.test.ts+++ b/skyvern-frontend/src/routes/workflows/editor/utils.test.ts@@ -5,6 +5,7 @@ import {WorkflowParameterValueType,} from "../types/workflowTypes";import {+ applySkyvernCredentialEdit,getInitialParameters,skyvernCredentialToParameterYAML,} from "./utils";@@ -81,6 +82,8 @@ describe("getInitialParameters", () => {key: "static_credential",description: null,credential_id: "cred_static",+ fallback_credential_ids: ["cred_backup"],+ fallback_trigger: "any_failure",created_at: "2026-06-23T00:00:00Z",modified_at: "2026-06-23T00:00:00Z",deleted_at: null,@@ -89,17 +92,21 @@ describe("getInitialParameters", () => {},};- expect(getInitialParameters(workflow)).toEqual([+ const initialParameters = getInitialParameters(workflow);++ expect(initialParameters).toEqual([expect.objectContaining({key: "credential_id",parameterType: "credential",dataType: WorkflowParameterValueType.CredentialId,}),- expect.not.objectContaining({+ expect.objectContaining({key: "static_credential",- dataType: WorkflowParameterValueType.CredentialId,+ fallbackCredentialIds: ["cred_backup"],+ fallbackTrigger: "any_failure",}),]);+ expect(initialParameters[1]).not.toHaveProperty("dataType");});});@@ -151,8 +158,148 @@ describe("skyvernCredentialToParameterYAML", () => {credential_id: "cred_1",credential_ids: ["cred_1", "cred_2"],selection_strategy: "round_robin",+ fallback_credential_ids: null,+ fallback_trigger: null,+ key: "credentials",+ description: null,+ });+ });++ test("serializes a single credential with fallbacks as a credential parameter", () => {+ const parameter: SkyvernCredential = {+ key: "credentials",+ parameterType: "credential",+ credentialId: "cred_1",+ fallbackCredentialIds: ["cred_backup"],+ fallbackTrigger: "credential_failures",+ };++ expect(skyvernCredentialToParameterYAML(parameter)).toEqual({+ parameter_type: "credential",+ credential_id: "cred_1",+ credential_ids: null,+ selection_strategy: null,+ fallback_credential_ids: ["cred_backup"],+ fallback_trigger: "credential_failures",key: "credentials",description: null,});});});++describe("applySkyvernCredentialEdit", () => {+ const previous: SkyvernCredential = {+ key: "portal_credential",+ parameterType: "credential",+ credentialId: "cred_primary",+ fallbackCredentialIds: ["cred_backup_1", "cred_backup_2"],+ fallbackTrigger: "any_failure",+ dataType: WorkflowParameterValueType.CredentialId,+ };++ test("preserves fallback config on an edit that only touches visible fields", () => {+ expect(+ applySkyvernCredentialEdit(previous, {+ key: "portal_credential",+ credentialId: "cred_primary",+ description: "updated description",+ }),+ ).toEqual({+ key: "portal_credential",+ parameterType: "credential",+ credentialId: "cred_primary",+ description: "updated description",+ credentialIds: null,+ selectionStrategy: null,+ fallbackCredentialIds: ["cred_backup_1", "cred_backup_2"],+ fallbackTrigger: "any_failure",+ dataType: WorkflowParameterValueType.CredentialId,+ });+ });++ test("removes the new primary from the fallback list on a primary change", () => {+ const edited = applySkyvernCredentialEdit(previous, {+ key: "portal_credential",+ credentialId: "cred_backup_1",+ });++ expect(edited.credentialId).toBe("cred_backup_1");+ expect(edited.fallbackCredentialIds).toEqual(["cred_backup_2"]);+ expect(edited.fallbackTrigger).toBe("any_failure");+ });++ test("clears the trigger when the primary change empties the fallback list", () => {+ const edited = applySkyvernCredentialEdit(+ { ...previous, fallbackCredentialIds: ["cred_backup_1"] },+ { key: "portal_credential", credentialId: "cred_backup_1" },+ );++ expect(edited.fallbackCredentialIds).toBeNull();+ expect(edited.fallbackTrigger).toBeNull();+ });++ test("preserves a rotation pool, swapping the new primary into its head", () => {+ const edited = applySkyvernCredentialEdit(+ {+ ...previous,+ credentialIds: ["cred_primary", "cred_rotate_1", "cred_rotate_2"],+ selectionStrategy: "round_robin",+ },+ { key: "portal_credential", credentialId: "cred_new" },+ );++ expect(edited.credentialIds).toEqual([+ "cred_new",+ "cred_rotate_1",+ "cred_rotate_2",+ ]);+ expect(edited.selectionStrategy).toBe("round_robin");+ });++ test("collapses a rotation pool that shrinks below two credentials", () => {+ const edited = applySkyvernCredentialEdit(+ {+ ...previous,+ credentialIds: ["cred_primary", "cred_rotate_1"],+ selectionStrategy: "round_robin",+ },+ { key: "portal_credential", credentialId: "cred_rotate_1" },+ );++ expect(edited.credentialIds).toBeNull();+ expect(edited.selectionStrategy).toBeNull();+ });++ test("returns a plain credential parameter when adding (no previous value)", () => {+ expect(+ applySkyvernCredentialEdit(undefined, {+ key: "credentials",+ credentialId: "cred_1",+ }),+ ).toEqual({+ key: "credentials",+ parameterType: "credential",+ credentialId: "cred_1",+ description: null,+ });+ });++ test("returns a plain credential parameter when the previous value is not a Skyvern credential", () => {+ expect(+ applySkyvernCredentialEdit(+ {+ key: "some_input",+ parameterType: "workflow",+ dataType: WorkflowParameterValueType.String,+ defaultValue: "x",+ },+ { key: "some_input", credentialId: "cred_1" },+ ),+ ).toEqual({+ key: "some_input",+ parameterType: "credential",+ credentialId: "cred_1",+ description: null,+ });+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/utils.ts b/skyvern-frontend/src/routes/workflows/editor/utils.tsindex 121de56b1..7e164559d 100644--- a/skyvern-frontend/src/routes/workflows/editor/utils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/utils.ts@@ -10,7 +10,11 @@ import {CredentialParameterYAML,WorkflowParameterYAML,} from "../types/workflowYamlTypes";-import { ParametersState, SkyvernCredential } from "./types";+import {+ ParametersState,+ parameterIsSkyvernCredential,+ SkyvernCredential,+} from "./types";/*** A single credential serializes to an editable workflow `credential_id`@@ -21,7 +25,9 @@ function skyvernCredentialToParameterYAML(parameter: SkyvernCredential,): WorkflowParameterYAML | CredentialParameterYAML {const hasCredentialRotation = (parameter.credentialIds?.length ?? 0) >= 2;- if (!hasCredentialRotation) {+ const hasFallbackCredentials =+ (parameter.fallbackCredentialIds?.length ?? 0) > 0;+ if (!hasCredentialRotation && !hasFallbackCredentials) {return {parameter_type: WorkflowParameterTypes.Workflow,workflow_parameter_type: WorkflowParameterValueType.CredentialId,@@ -35,11 +41,67 @@ function skyvernCredentialToParameterYAML(credential_id: parameter.credentialId,credential_ids: parameter.credentialIds ?? null,selection_strategy: parameter.selectionStrategy ?? null,+ fallback_credential_ids: parameter.fallbackCredentialIds ?? null,+ fallback_trigger: parameter.fallbackTrigger ?? null,key: parameter.key,description: parameter.description || null,};}+/**+ * The parameter edit panel only exposes key/credential/description, so an edit+ * must carry over the rotation pool and fallback config it can't see — a plain+ * rebuild silently wipes them. When the primary credential changes it is+ * removed from the fallback list (a credential may not be its own fallback)+ * and swapped into the head of the rotation pool.+ */+function applySkyvernCredentialEdit(+ previous: ParametersState[number] | null | undefined,+ edit: { key: string; credentialId: string; description?: string | null },+): SkyvernCredential {+ const base: SkyvernCredential = {+ key: edit.key,+ parameterType: "credential",+ credentialId: edit.credentialId,+ description: edit.description ?? null,+ };+ if (+ !previous ||+ previous.parameterType !== "credential" ||+ !parameterIsSkyvernCredential(previous)+ ) {+ return base;+ }++ const previousPool = previous.credentialIds ?? [];+ const rotationPool =+ previousPool.length >= 2+ ? [+ edit.credentialId,+ ...previousPool.filter(+ (id) => id !== previous.credentialId && id !== edit.credentialId,+ ),+ ]+ : [];+ const credentialIds = rotationPool.length >= 2 ? rotationPool : null;++ const fallbackCredentialIds = (previous.fallbackCredentialIds ?? []).filter(+ (id) => id !== edit.credentialId,+ );+ const hasFallback = fallbackCredentialIds.length > 0;++ return {+ ...base,+ credentialIds,+ selectionStrategy: credentialIds+ ? (previous.selectionStrategy ?? null)+ : null,+ fallbackCredentialIds: hasFallback ? fallbackCredentialIds : null,+ fallbackTrigger: hasFallback ? (previous.fallbackTrigger ?? null) : null,+ dataType: previous.dataType,+ };+}+const getInitialParameters = (workflow: WorkflowApiResponse) => {return workflow.workflow_definition.parameters.filter((parameter) => isDisplayedInWorkflowEditor(parameter))@@ -103,6 +165,8 @@ const getInitialParameters = (workflow: WorkflowApiResponse) => {credentialId: parameter.credential_id,credentialIds: parameter.credential_ids ?? null,selectionStrategy: parameter.selection_strategy ?? null,+ fallbackCredentialIds: parameter.fallback_credential_ids ?? null,+ fallbackTrigger: parameter.fallback_trigger ?? null,description: parameter.description,};} else if (@@ -202,6 +266,7 @@ const constructCacheKeyValueFromParameters = (opts: {};export {+ applySkyvernCredentialEdit,constructCacheKeyValue,constructCacheKeyValueFromParameters,getInitialParameters,diff --git a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.tsindex 18d04da8f..d6f67a9b1 100644--- a/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts+++ b/skyvern-frontend/src/routes/workflows/editor/workflowEditorUtils.ts@@ -4253,6 +4253,8 @@ function convertParametersToParameterYAML(credential_id: parameter.credential_id,credential_ids: parameter.credential_ids ?? null,selection_strategy: parameter.selection_strategy ?? null,+ fallback_credential_ids: parameter.fallback_credential_ids ?? null,+ fallback_trigger: parameter.fallback_trigger ?? null,};}case WorkflowParameterTypes.OnePassword: {diff --git a/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.test.ts b/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.test.tsindex bc44848aa..b16c05ed7 100644--- a/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.test.ts+++ b/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.test.ts@@ -8,7 +8,11 @@ import {WorkflowParameterTypes,WorkflowParameterValueType,} from "./types/workflowTypes";-import { getLoginCredentialInputs } from "./runWorkflowCredentials";+import {+ getFallbackCredentialIds,+ getLoginCredentialInputs,+ getRotatingCredentialIds,+} from "./runWorkflowCredentials";function workflowParameter(key: string): WorkflowParameter {return {@@ -80,6 +84,8 @@ describe("getLoginCredentialInputs", () => {{parameter: loginCredential,loginBlockLabels: ["Portal login"],+ fallbackCredentialIds: [],+ fallbackTrigger: null,},]);});@@ -103,6 +109,8 @@ describe("getLoginCredentialInputs", () => {{parameter: credential,loginBlockLabels: ["Primary login", "Nested login"],+ fallbackCredentialIds: [],+ fallbackTrigger: null,},]);});@@ -122,8 +130,90 @@ describe("getLoginCredentialInputs", () => {{parameter: credential,loginBlockLabels: ["Rotating login"],+ fallbackCredentialIds: [],+ fallbackTrigger: null,+ },+ ]);+ });++ test("returns single-credential parameters that have fallback credentials", () => {+ const credential: CredentialParameter = {+ ...rotatingCredentialParameter("account_credential", ["cred_1"]),+ credential_ids: null,+ selection_strategy: null,+ fallback_credential_ids: ["cred_fb1"],+ };++ const inputs = getLoginCredentialInputs({+ workflow: workflowWithBlocks(+ [loginBlock("Fallback login", [credential])],+ [credential],+ ),+ workflowParameters: [],+ });++ expect(inputs).toEqual([+ {+ parameter: credential,+ loginBlockLabels: ["Fallback login"],+ fallbackCredentialIds: ["cred_fb1"],+ fallbackTrigger: "credential_failures",},]);+ expect((inputs[0]!.parameter as CredentialParameter).credential_id).toBe(+ "cred_1",+ );+ });++ test("passes ordered fallback credential ids through and defaults the trigger", () => {+ const credential: CredentialParameter = {+ ...rotatingCredentialParameter("account_credential"),+ fallback_credential_ids: ["cred_fb2", "cred_fb1", "cred_fb2"],+ };++ const inputs = getLoginCredentialInputs({+ workflow: workflowWithBlocks(+ [loginBlock("Rotating login", [credential])],+ [credential],+ ),+ workflowParameters: [],+ });++ expect(inputs[0]!.fallbackCredentialIds).toEqual(["cred_fb2", "cred_fb1"]);+ expect(inputs[0]!.fallbackTrigger).toBe("credential_failures");+ });++ test("preserves an explicit any_failure fallback trigger", () => {+ const credential: CredentialParameter = {+ ...rotatingCredentialParameter("account_credential"),+ fallback_credential_ids: ["cred_fb1"],+ fallback_trigger: "any_failure",+ };++ const inputs = getLoginCredentialInputs({+ workflow: workflowWithBlocks(+ [loginBlock("Rotating login", [credential])],+ [credential],+ ),+ workflowParameters: [],+ });++ expect(inputs[0]!.fallbackTrigger).toBe("any_failure");+ });++ test("classifies fallback-only parameters as single-credential, not rotating", () => {+ const credential: CredentialParameter = {+ ...rotatingCredentialParameter("account_credential", ["cred_1"]),+ credential_ids: null,+ selection_strategy: null,+ fallback_credential_ids: ["cred_fb1", "cred_fb2"],+ };++ expect(getRotatingCredentialIds(credential)).toEqual([]);+ expect(getFallbackCredentialIds(credential)).toEqual([+ "cred_fb1",+ "cred_fb2",+ ]);});test("ignores block-scoped single credentials", () => {@@ -149,3 +239,40 @@ describe("getLoginCredentialInputs", () => {expect(inputs).toEqual([]);});});++describe("getFallbackCredentialIds", () => {+ test("deduplicates fallback credential ids while preserving order", () => {+ const credential = {+ ...rotatingCredentialParameter("account_credential"),+ fallback_credential_ids: ["fallback_1", "fallback_2", "fallback_1"],+ };++ expect(getFallbackCredentialIds(credential)).toEqual([+ "fallback_1",+ "fallback_2",+ ]);+ });++ test("ignores empty and invalid fallback credential ids", () => {+ const credential = {+ ...rotatingCredentialParameter("account_credential"),+ fallback_credential_ids: [+ "fallback_1",+ "",+ null,+ "fallback_2",+ ] as unknown as Array<string>,+ };++ expect(getFallbackCredentialIds(credential)).toEqual([+ "fallback_1",+ "fallback_2",+ ]);+ });++ test("returns an empty list when fallback credentials are unset", () => {+ const credential = rotatingCredentialParameter("account_credential");++ expect(getFallbackCredentialIds(credential)).toEqual([]);+ });+});diff --git a/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.ts b/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.tsindex 62d8eba99..fa7a9e41d 100644--- a/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.ts+++ b/skyvern-frontend/src/routes/workflows/runWorkflowCredentials.ts@@ -1,4 +1,5 @@import {+ CredentialFallbackTrigger,CredentialParameter,WorkflowApiResponse,WorkflowParameter,@@ -10,6 +11,8 @@ import { visitWorkflowBlocks } from "./workflowBlockUtils";export type LoginCredentialInput = {parameter: WorkflowParameter | CredentialParameter;loginBlockLabels: Array<string>;+ fallbackCredentialIds: Array<string>;+ fallbackTrigger: CredentialFallbackTrigger | null;};function isCredentialIdWorkflowParameter(@@ -39,7 +42,18 @@ export function getRotatingCredentialIds();}-function isRotatingCredentialParameter(+export function getFallbackCredentialIds(+ parameter: CredentialParameter,+): Array<string> {+ return (parameter.fallback_credential_ids ?? []).filter(+ (credentialId, index, allCredentialIds): credentialId is string =>+ typeof credentialId === "string" &&+ credentialId.length > 0 &&+ allCredentialIds.indexOf(credentialId) === index,+ );+}++function isConfigurableCredentialParameter(parameter: unknown,): parameter is CredentialParameter {if (parameter === null || typeof parameter !== "object") {@@ -47,10 +61,17 @@ function isRotatingCredentialParameter(}const maybeParameter = parameter as Partial<CredentialParameter>;+ if (+ maybeParameter.parameter_type !== WorkflowParameterTypes.Credential ||+ typeof maybeParameter.key !== "string"+ ) {+ return false;+ }++ const credentialParameter = maybeParameter as CredentialParameter;return (- maybeParameter.parameter_type === WorkflowParameterTypes.Credential &&- typeof maybeParameter.key === "string" &&- getRotatingCredentialIds(maybeParameter as CredentialParameter).length > 1+ getRotatingCredentialIds(credentialParameter).length > 1 ||+ getFallbackCredentialIds(credentialParameter).length > 0);}@@ -68,7 +89,7 @@ function collectLoginCredentialLabels(for (const parameter of block.parameters ?? []) {if (!isCredentialIdWorkflowParameter(parameter) &&- !isRotatingCredentialParameter(parameter)+ !isConfigurableCredentialParameter(parameter)) {continue;}@@ -102,7 +123,7 @@ export function getLoginCredentialInputs({[...workflowParameters.filter(isCredentialIdWorkflowParameter),...workflow.workflow_definition.parameters.filter(- isRotatingCredentialParameter,+ isConfigurableCredentialParameter,),].map((parameter) => [parameter.key, parameter]),);@@ -123,10 +144,22 @@ export function getLoginCredentialInputs({return [];}+ const fallbackCredentialIds =+ parameter.parameter_type === WorkflowParameterTypes.Credential+ ? getFallbackCredentialIds(parameter)+ : [];+ const fallbackTrigger =+ fallbackCredentialIds.length > 0 &&+ parameter.parameter_type === WorkflowParameterTypes.Credential+ ? (parameter.fallback_trigger ?? "credential_failures")+ : null;+return [{parameter,loginBlockLabels: Array.from(labels),+ fallbackCredentialIds,+ fallbackTrigger,},];});diff --git a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowTypes.tsindex 24aebfaf5..b8cfbcb54 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowTypes.ts@@ -86,6 +86,7 @@ export type AzureVaultCredentialParameter = WorkflowParameterBase & {};export type CredentialSelectionStrategy = "round_robin" | "random";+export type CredentialFallbackTrigger = "credential_failures" | "any_failure";export type CredentialParameter = WorkflowParameterBase & {parameter_type: "credential";@@ -94,6 +95,8 @@ export type CredentialParameter = WorkflowParameterBase & {credential_id: string;credential_ids?: Array<string> | null;selection_strategy?: CredentialSelectionStrategy | null;+ fallback_credential_ids?: Array<string> | null;+ fallback_trigger?: CredentialFallbackTrigger | null;created_at: string;modified_at: string;deleted_at: string | null;diff --git a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.tsindex c07cea96e..e9498c8eb 100644--- a/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts+++ b/skyvern-frontend/src/routes/workflows/types/workflowYamlTypes.ts@@ -1,6 +1,7 @@import { ProxyLocation, RunEngine } from "@/api/types";import {CodeBlockStep,+ CredentialFallbackTrigger,CredentialSelectionStrategy,WorkflowBlockType,WorkflowModel,@@ -133,6 +134,8 @@ export type CredentialParameterYAML = ParameterYAMLBase & {credential_id: string;credential_ids?: Array<string> | null;selection_strategy?: CredentialSelectionStrategy | null;+ fallback_credential_ids?: Array<string> | null;+ fallback_trigger?: CredentialFallbackTrigger | null;};export type BlockYAML =diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 9c3d02ea8..17fa33e35 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -719,6 +719,12 @@ class AgentFunction:def has_proxy_session_extra_http_headers(self, extra_http_headers: dict[str, str] | None) -> bool:return False+ def strip_proxy_session_extra_http_headers(+ self,+ extra_http_headers: dict[str, str] | None,+ ) -> dict[str, str] | None:+ return extra_http_headers+def merge_proxy_session_extra_http_headers(self,extra_http_headers: dict[str, str] | None,diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex e6bb93103..139a4ca45 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -700,6 +700,13 @@ class WorkflowRunModel(Base):"queued_at",postgresql_where=text("status IN ('queued', 'running', 'paused') AND browser_session_id IS NULL"),),+ Index(+ "ix_workflow_runs_retried_from_workflow_run_id",+ "retried_from_workflow_run_id",+ unique=True,+ postgresql_where=text("retried_from_workflow_run_id IS NOT NULL"),+ sqlite_where=text("retried_from_workflow_run_id IS NOT NULL"),+ ),)workflow_run_id = Column(String, primary_key=True, default=generate_workflow_run_id)@@ -730,6 +737,8 @@ class WorkflowRunModel(Base):debug_session_id: Column = Column(String, nullable=True)trigger_type = Column(String, nullable=True)workflow_schedule_id = Column(String, nullable=True, index=True)+ retried_from_workflow_run_id = Column(String, nullable=True)+ fallback_attempt = Column(Integer, nullable=True)ai_fallback = Column(Boolean, nullable=True)code_gen = Column(Boolean, nullable=True)waiting_for_verification_code = Column(Boolean, nullable=False, default=False, server_default=sqlalchemy.false())@@ -908,6 +917,8 @@ class CredentialParameterModel(Base):credential_id = Column(String, nullable=False)credential_ids = Column(JSON, nullable=True)selection_strategy = Column(String, nullable=True)+ fallback_credential_ids = Column(JSON, nullable=True)+ fallback_trigger = Column(String, nullable=True)created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False)diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.pyindex 2bca94328..c1254e69e 100644--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py@@ -292,6 +292,8 @@ class WorkflowParametersRepository(BaseRepository):credential_id=parameter.credential_id,credential_ids=parameter.credential_ids,selection_strategy=parameter.selection_strategy,+ fallback_credential_ids=parameter.fallback_credential_ids,+ fallback_trigger=parameter.fallback_trigger,deleted_at=parameter.deleted_at,)elif isinstance(parameter, OnePasswordCredentialParameter):diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.pyindex 659894b81..abfc7dab3 100644--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py@@ -208,6 +208,8 @@ class WorkflowRunsRepository(BaseRepository):workflow_run_id: str | None = None,trigger_type: WorkflowRunTriggerType | None = None,workflow_schedule_id: str | None = None,+ retried_from_workflow_run_id: str | None = None,+ fallback_attempt: int | None = None,ignore_inherited_workflow_system_prompt: bool = False,copilot_session_id: str | None = None,) -> WorkflowRun:@@ -239,6 +241,8 @@ class WorkflowRunsRepository(BaseRepository):code_gen=code_gen,trigger_type=trigger_type.value if trigger_type else None,workflow_schedule_id=workflow_schedule_id,+ retried_from_workflow_run_id=retried_from_workflow_run_id,+ fallback_attempt=fallback_attempt,ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,copilot_session_id=copilot_session_id,**kwargs,@@ -248,6 +252,18 @@ class WorkflowRunsRepository(BaseRepository):await session.refresh(workflow_run)return convert_to_workflow_run(workflow_run)+ @db_operation("get_workflow_run_retried_by")+ async def get_workflow_run_retried_by(self, workflow_run_id: str, organization_id: str) -> str | None:+ async with self.Session() as session:+ query = (+ select(WorkflowRunModel.workflow_run_id)+ .where(WorkflowRunModel.retried_from_workflow_run_id == workflow_run_id)+ .where(WorkflowRunModel.organization_id == organization_id)+ .order_by(WorkflowRunModel.created_at.desc())+ .limit(1)+ )+ return (await session.scalars(query)).first()+@db_operation("update_workflow_run")async def update_workflow_run(self,diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex 4d5b86083..c91169186 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -613,6 +613,8 @@ def convert_to_workflow_run(browser_profile_id=workflow_run_model.browser_profile_id,status=WorkflowRunStatus[workflow_run_model.status],failure_reason=workflow_run_model.failure_reason,+ retried_from_workflow_run_id=workflow_run_model.retried_from_workflow_run_id,+ fallback_attempt=workflow_run_model.fallback_attempt,proxy_location=deserialize_proxy_location(workflow_run_model.proxy_location),webhook_callback_url=workflow_run_model.webhook_callback_url,webhook_failure_reason=workflow_run_model.webhook_failure_reason,diff --git a/skyvern/forge/sdk/routes/agent_protocol.py b/skyvern/forge/sdk/routes/agent_protocol.pyindex 6986e63c9..cd1c11699 100644--- a/skyvern/forge/sdk/routes/agent_protocol.py+++ b/skyvern/forge/sdk/routes/agent_protocol.py@@ -3582,30 +3582,6 @@ async def _continue_workflow_run(workflow_run_id: str, organization_id: str) ->await app.WORKFLOW_SERVICE.mark_workflow_run_as_running(workflow_run_id)-def _workflow_request_body_from_existing_run(- workflow_run: WorkflowRun,- parameters: dict[str, Any] | None = None,- run_metadata: dict[str, str] | None = None,-) -> WorkflowRequestBody:- return WorkflowRequestBody(- data=parameters,- proxy_location=workflow_run.proxy_location,- webhook_callback_url=workflow_run.webhook_callback_url,- totp_verification_url=workflow_run.totp_verification_url,- totp_identifier=workflow_run.totp_identifier,- browser_session_id=workflow_run.browser_session_id,- browser_profile_id=workflow_run.browser_profile_id,- max_screenshot_scrolls=workflow_run.max_screenshot_scrolls,- max_elapsed_time_minutes=getattr(workflow_run, "max_elapsed_time_minutes", None),- extra_http_headers=workflow_run.extra_http_headers,- cdp_connect_headers=workflow_run.cdp_connect_headers,- browser_address=workflow_run.browser_address,- run_with=workflow_run.run_with,- ai_fallback=workflow_run.ai_fallback,- run_metadata=run_metadata,- )--def _workflow_run_request_from_workflow_request(*,workflow_id: str,@@ -3736,7 +3712,7 @@ async def retry_workflow_run(organization_id=current_org.organization_id,exc_info=True,)- legacy_workflow_request = _workflow_request_body_from_existing_run(+ legacy_workflow_request = workflow_service.workflow_request_body_from_existing_run(workflow_run=original_workflow_run,parameters=original_workflow_run_parameters,run_metadata=original_run_metadata,diff --git a/skyvern/forge/sdk/workflow/context_manager.py b/skyvern/forge/sdk/workflow/context_manager.pyindex f42c94de8..9c5a79d75 100644--- a/skyvern/forge/sdk/workflow/context_manager.py+++ b/skyvern/forge/sdk/workflow/context_manager.py@@ -722,7 +722,7 @@ class WorkflowRunContext:LOG.info("Fetching credential parameter value", parameter_key=parameter.key)credential_id = None- if parameter.credential_ids:+ if parameter.credential_ids or parameter.fallback_credential_ids:credential_id = await self.resolve_credential_parameter_id(parameter, organization.organization_id)elif parameter.credential_id:if self.has_parameter(parameter.credential_id) and self.has_value(parameter.credential_id):@@ -745,8 +745,18 @@ class WorkflowRunContext:if cached:return cachedif not parameter.credential_ids:- self.resolved_credential_parameter_ids[parameter.key] = parameter.credential_id- return parameter.credential_id+ credential_id = parameter.credential_id+ if parameter.fallback_credential_ids:+ selected = await app.DATABASE.workflow_run_credential_selections.get_selection(+ workflow_run_id=self.workflow_run_id,+ parameter_key=parameter.key,+ )+ if selected:+ credential_id = selected+ elif self.has_parameter(credential_id) and self.has_value(credential_id):+ credential_id = self.values[credential_id]+ self.resolved_credential_parameter_ids[parameter.key] = credential_id+ return credential_idcredential_id = await select_credential_for_run(workflow_run_id=self.workflow_run_id,organization_id=organization_id,diff --git a/skyvern/forge/sdk/workflow/credential_fallback.py b/skyvern/forge/sdk/workflow/credential_fallback.pynew file mode 100644index 000000000..c7d5b8f5d--- /dev/null+++ b/skyvern/forge/sdk/workflow/credential_fallback.py@@ -0,0 +1,255 @@+from __future__ import annotations++import structlog+from sqlalchemy.exc import IntegrityError++from skyvern.exceptions import WorkflowNotFound+from skyvern.forge import app+from skyvern.forge.failure_classifier import classify_from_failure_reason+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus++LOG = structlog.get_logger()++CREDENTIAL_FAILURES = "credential_failures"+ANY_FAILURE = "any_failure"+VALID_FALLBACK_TRIGGERS = frozenset({CREDENTIAL_FAILURES, ANY_FAILURE})+CREDENTIAL_FAILURE_CATEGORIES = frozenset({"AUTH_FAILURE", "CREDENTIAL_ERROR"})+CREDENTIAL_FALLBACK_RETRY_FLAG = "CREDENTIAL_FALLBACK_RETRY"+++async def _retry_enabled_for_organization(organization_id: str, workflow_run: WorkflowRun) -> bool:+ """Gate the auto-retry rollout. Fails closed: a retry run costs credits and re-fires webhooks.++ distinct_id = organization_id (deliberately NOT run-level). backend.md's per-run+ randomization convention serves per-request experiments; retry behavior must be+ deterministic per customer, so a given org is fully in-or-out rather than retrying+ on a random subset of its runs. Do not "fix" this to workflow_run_id.+ """+ provider = getattr(app, "EXPERIMENTATION_PROVIDER", None)+ if not provider:+ return False+ try:+ return bool(+ await provider.is_feature_enabled_cached(+ CREDENTIAL_FALLBACK_RETRY_FLAG,+ organization_id,+ properties={+ "organization_id": organization_id,+ "workflow_permanent_id": workflow_run.workflow_permanent_id,+ },+ )+ )+ except Exception:+ LOG.warning(+ "credential_fallback.flag_error",+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ exc_info=True,+ )+ return False+++def _failure_category_names(workflow_run: WorkflowRun) -> set[str]:+ categories = workflow_run.failure_category+ if not categories:+ categories = classify_from_failure_reason(workflow_run.failure_reason)+ return {+ category+ for entry in categories or []+ if isinstance(entry, dict) and isinstance(category := entry.get("category"), str)+ }+++def _trigger_matches(workflow_run: WorkflowRun, trigger: str | None) -> bool:+ if workflow_run.status not in {WorkflowRunStatus.failed, WorkflowRunStatus.terminated}:+ return False+ effective_trigger = trigger or CREDENTIAL_FAILURES+ if effective_trigger == ANY_FAILURE:+ return True+ if effective_trigger != CREDENTIAL_FAILURES:+ return False+ return bool(_failure_category_names(workflow_run) & CREDENTIAL_FAILURE_CATEGORIES)+++async def maybe_start_credential_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:+ try:+ return await _maybe_start_credential_fallback_retry(workflow_run, organization_id)+ except Exception:+ LOG.warning(+ "Failed to start credential fallback retry",+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ exc_info=True,+ )+ return None+++async def _maybe_start_credential_fallback_retry(workflow_run: WorkflowRun, organization_id: str) -> str | None:+ if workflow_run.parent_workflow_run_id or workflow_run.debug_session_id:+ return None++ if not await _retry_enabled_for_organization(organization_id, workflow_run):+ return None++ is_block_scoped_run = await app.AGENT_FUNCTION.is_block_scoped_workflow_run(workflow_run)+ if not is_block_scoped_run:+ is_block_scoped_run = await app.DATABASE.debug.has_block_run_for_workflow_run(+ organization_id=organization_id,+ workflow_run_id=workflow_run.workflow_run_id,+ )+ if is_block_scoped_run:+ return None++ try:+ workflow = await app.WORKFLOW_SERVICE.get_workflow(+ workflow_id=workflow_run.workflow_id,+ organization_id=organization_id,+ )+ except WorkflowNotFound:+ return None+ if workflow is None:+ return None++ credential_parameters = [+ parameter+ for parameter in workflow.workflow_definition.parameters+ if isinstance(parameter, CredentialParameter) and bool(parameter.fallback_credential_ids)+ ]+ if not credential_parameters:+ return None++ existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ )+ if existing_retry_run_id:+ return existing_retry_run_id++ prior_selections = await app.DATABASE.workflow_run_credential_selections.get_selections_for_run(+ workflow_run.workflow_run_id+ )++ next_attempt = (workflow_run.fallback_attempt or 0) + 1+ triggered_parameters = [+ parameter for parameter in credential_parameters if _trigger_matches(workflow_run, parameter.fallback_trigger)+ ]+ if not triggered_parameters:+ return None++ all_fallback_ids = list(+ dict.fromkeys(+ credential_id+ for parameter in triggered_parameters+ for credential_id in parameter.fallback_credential_ids or []+ )+ )+ existing_credentials = await app.DATABASE.credentials.get_credentials_by_ids(+ all_fallback_ids, organization_id=organization_id+ )+ found_credential_ids = {credential.credential_id for credential in existing_credentials}+ missing_credential_ids = [+ credential_id for credential_id in all_fallback_ids if credential_id not in found_credential_ids+ ]+ if missing_credential_ids:+ LOG.warning(+ "Skipping credential fallback candidates whose credentials no longer exist",+ workflow_run_id=workflow_run.workflow_run_id,+ missing_credential_ids=missing_credential_ids,+ )++ overrides: dict[str, str] = {}+ for parameter in triggered_parameters:+ fallback_ids = parameter.fallback_credential_ids or []+ prior = prior_selections.get(parameter.key)+ # A prior selection only marks fallback progress on a fallback retry; on an original+ # run it may come from rotation and coincidentally appear in the fallback list.+ base_index = fallback_ids.index(prior) + 1 if workflow_run.fallback_attempt and prior in fallback_ids else 0+ candidate = next(+ (+ credential_id+ for credential_id in fallback_ids[base_index:]+ if credential_id in found_credential_ids and credential_id != prior+ ),+ None,+ )+ if candidate is not None:+ overrides[parameter.key] = candidate+ if not overrides:+ return None++ advanced_parameter_keys = list(overrides)+ credential_parameter_keys = {+ parameter.key+ for parameter in workflow.workflow_definition.parameters+ if isinstance(parameter, CredentialParameter)+ }+ for parameter_key, credential_id in prior_selections.items():+ if parameter_key in credential_parameter_keys and parameter_key not in overrides:+ overrides[parameter_key] = credential_id++ parameter_tuples = await app.DATABASE.workflow_runs.get_workflow_run_parameters(+ workflow_run_id=workflow_run.workflow_run_id,+ )+ original_parameters = {+ workflow_parameter.key: workflow_run_parameter.value+ for workflow_parameter, workflow_run_parameter in parameter_tuples+ }+ retry_parameters = {**original_parameters, **overrides}++ organization = await app.DATABASE.organizations.get_organization(organization_id)+ if organization is None:+ return None++ from skyvern.services.workflow_service import run_workflow, workflow_request_body_from_existing_run++ workflow_request = workflow_request_body_from_existing_run(+ workflow_run=workflow_run,+ parameters=retry_parameters,+ )+ workflow_request.browser_session_id = None+ workflow_request.browser_profile_id = None+ workflow_request.extra_http_headers = app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers(+ workflow_request.extra_http_headers+ )+ try:+ # A fresh context keeps run creation (and the background execution task it spawns,+ # which inherits contextvars) from attributing the retry to the failed run.+ with skyvern_context.scoped(SkyvernContext()):+ retry_run = await run_workflow(+ workflow_id=workflow_run.workflow_permanent_id,+ organization=organization,+ workflow_request=workflow_request,+ template=False,+ version=workflow.version,+ trigger_type=workflow_run.trigger_type,+ workflow_schedule_id=workflow_run.workflow_schedule_id,+ retried_from_workflow_run_id=workflow_run.workflow_run_id,+ fallback_attempt=next_attempt,+ ignore_inherited_workflow_system_prompt=workflow_run.ignore_inherited_workflow_system_prompt,+ )+ except IntegrityError as exc:+ error_text = str(exc.orig) if exc.orig is not None else str(exc)+ if (+ "ix_workflow_runs_retried_from_workflow_run_id" not in error_text+ and "retried_from_workflow_run_id" not in error_text+ ):+ raise+ existing_retry_run_id = await app.DATABASE.workflow_runs.get_workflow_run_retried_by(+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=organization_id,+ )+ if not existing_retry_run_id:+ raise+ return existing_retry_run_id++ LOG.info(+ "Started credential fallback retry",+ workflow_run_id=workflow_run.workflow_run_id,+ new_run_id=retry_run.workflow_run_id,+ fallback_attempt=next_attempt,+ advanced_parameter_keys=advanced_parameter_keys,+ )+ return retry_run.workflow_run_iddiff --git a/skyvern/forge/sdk/workflow/models/parameter.py b/skyvern/forge/sdk/workflow/models/parameter.pyindex 0ce962523..d7b83bbd1 100644--- a/skyvern/forge/sdk/workflow/models/parameter.py+++ b/skyvern/forge/sdk/workflow/models/parameter.py@@ -127,6 +127,8 @@ class CredentialParameter(Parameter):credential_id: strcredential_ids: list[str] | None = Noneselection_strategy: str | None = None+ fallback_credential_ids: list[str] | None = None+ fallback_trigger: str | None = Nonecreated_at: datetimemodified_at: datetimediff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.pyindex ee5d91399..eda399209 100644--- a/skyvern/forge/sdk/workflow/models/workflow.py+++ b/skyvern/forge/sdk/workflow/models/workflow.py@@ -239,6 +239,8 @@ class WorkflowRun(BaseModel):totp_identifier: str | None = Nonefailure_reason: str | None = Nonefailure_category: list[dict[str, Any]] | None = None+ retried_from_workflow_run_id: str | None = None+ fallback_attempt: int | None = Noneparent_workflow_run_id: str | None = Noneworkflow_title: str | None = Nonemax_screenshot_scrolls: int | None = None@@ -346,6 +348,8 @@ class WorkflowRunResponseBase(BaseModel):status: WorkflowRunStatusfailure_reason: str | None = Nonefailure_category: list[dict[str, Any]] | None = None+ retried_from_workflow_run_id: str | None = None+ retried_by_workflow_run_id: str | None = Noneproxy_location: ProxyLocationInput = Nonewebhook_callback_url: str | None = Nonewebhook_failure_reason: str | None = Nonediff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex d2cc3dbe7..92a34b7b8 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -83,6 +83,10 @@ from skyvern.forge.sdk.workflow.browser_profile_key import (build_workflow_browser_session_storage_key,render_browser_profile_key,)+from skyvern.forge.sdk.workflow.credential_fallback import (+ VALID_FALLBACK_TRIGGERS,+ maybe_start_credential_fallback_retry,+)from skyvern.forge.sdk.workflow.credential_selection import (VALID_SELECTION_STRATEGIES,normalize_selection_strategy,@@ -732,6 +736,23 @@ class WorkflowService:exc_info=True,)+ @staticmethod+ async def _start_credential_fallback_retry_best_effort(workflow_run: WorkflowRun) -> None:+ try:+ await maybe_start_credential_fallback_retry(workflow_run, workflow_run.organization_id)+ except Exception:+ LOG.warning(+ "Credential fallback retry hook failed",+ workflow_run_id=workflow_run.workflow_run_id,+ organization_id=workflow_run.organization_id,+ exc_info=True,+ )++ def _schedule_credential_fallback_retry(self, workflow_run: WorkflowRun) -> None:+ task = asyncio.create_task(self._start_credential_fallback_retry_best_effort(workflow_run))+ self._background_tasks.add(task)+ task.add_done_callback(self._background_tasks.discard)+@staticmethoddef _determine_cache_invalidation(previous_blocks: list[dict[str, Any]],@@ -1138,28 +1159,67 @@ class WorkflowService:) -> None:credential_ids_to_validate: list[str] = []for parameter in parameters:- credential_ids = getattr(parameter, "credential_ids", None)- if credential_ids is None:- continuekey = getattr(parameter, "key", None) or "<unknown>"- if not credential_ids:+ credential_ids = getattr(parameter, "credential_ids", None)+ if credential_ids is not None:+ if not credential_ids:+ raise SkyvernHTTPException(+ message=f"credential_ids for credential parameter {key} must be non-empty.",+ status_code=400,+ )+ credential_ids = list(dict.fromkeys(credential_ids))+ parameter.credential_ids = credential_ids+ selection_strategy = getattr(parameter, "selection_strategy", None)+ if normalize_selection_strategy(selection_strategy) not in VALID_SELECTION_STRATEGIES:+ raise SkyvernHTTPException(+ message=(+ f"selection_strategy for credential parameter {key} must be one of: "+ f"{', '.join(sorted(VALID_SELECTION_STRATEGIES))}."+ ),+ status_code=400,+ )+ parameter.credential_id = credential_ids[0]+ credential_ids_to_validate.extend(credential_ids)++ fallback_credential_ids = getattr(parameter, "fallback_credential_ids", None)+ if fallback_credential_ids is not None:+ fallback_credential_ids = list(dict.fromkeys(fallback_credential_ids))+ if credential_ids is None:+ primary_credential_id = getattr(parameter, "credential_id", None)+ fallback_credential_ids = [+ credential_id+ for credential_id in fallback_credential_ids+ if credential_id != primary_credential_id+ ]+ parameter.fallback_credential_ids = fallback_credential_ids or None+ credential_ids_to_validate.extend(fallback_credential_ids)++ if (+ getattr(parameter, "credential_ids", None) is not None+ and getattr(parameter, "fallback_credential_ids", None) is not None+ ):raise SkyvernHTTPException(- message=f"credential_ids for credential parameter {key} must be non-empty.",+ message=(+ f"credential parameter {key} cannot combine credential_ids rotation with "+ "fallback_credential_ids; configure one or the other."+ ),status_code=400,)- credential_ids = list(dict.fromkeys(credential_ids))- parameter.credential_ids = credential_ids- selection_strategy = getattr(parameter, "selection_strategy", None)- if normalize_selection_strategy(selection_strategy) not in VALID_SELECTION_STRATEGIES:++ fallback_trigger = getattr(parameter, "fallback_trigger", None)+ if fallback_trigger is not None and fallback_trigger not in VALID_FALLBACK_TRIGGERS:raise SkyvernHTTPException(message=(- f"selection_strategy for credential parameter {key} must be one of: "- f"{', '.join(sorted(VALID_SELECTION_STRATEGIES))}."+ f"fallback_trigger for credential parameter {key} must be one of: "+ f"{', '.join(sorted(VALID_FALLBACK_TRIGGERS))}."),status_code=400,)- parameter.credential_id = credential_ids[0]- credential_ids_to_validate.extend(credential_ids)+ if fallback_trigger is not None and not getattr(parameter, "fallback_credential_ids", None):+ raise SkyvernHTTPException(+ message=f"fallback_trigger for credential parameter {key} requires fallback_credential_ids.",+ status_code=400,+ )await self._validate_credential_ids(credential_ids_to_validate, organization)@@ -1174,6 +1234,18 @@ class WorkflowService:if isinstance(parameter, CredentialParameter) and bool(parameter.credential_ids)]+ @staticmethod+ def _get_credential_parameters_with_configured_selection(workflow: Workflow) -> list[CredentialParameter]:+ workflow_definition = getattr(workflow, "workflow_definition", None)+ if workflow_definition is None:+ return []+ return [+ parameter+ for parameter in workflow_definition.parameters+ if isinstance(parameter, CredentialParameter)+ and (bool(parameter.credential_ids) or bool(parameter.fallback_credential_ids))+ ]+def _get_run_credential_parameter_overrides(self,*,@@ -1184,7 +1256,7 @@ class WorkflowService:return {}overrides: dict[str, str] = {}- for parameter in self._get_rotating_credential_parameters(workflow):+ for parameter in self._get_credential_parameters_with_configured_selection(workflow):if parameter.key not in request_data:continue@@ -1194,12 +1266,18 @@ class WorkflowService:if not isinstance(override, str):raise InvalidCredentialId(f"<non-string value of type {type(override).__name__}>")- credential_ids = parameter.credential_ids or []+ credential_ids = list(+ dict.fromkeys(+ (parameter.credential_ids or [])+ + (parameter.fallback_credential_ids or [])+ + [parameter.credential_id]+ )+ )if override not in credential_ids:raise SkyvernHTTPException(message=(f"Credential override for parameter {parameter.key} must be one of the configured "- "rotation credentials."+ "rotation or fallback credentials."),status_code=400,)@@ -1370,6 +1448,8 @@ class WorkflowService:workflow_run_id: str | None = None,trigger_type: WorkflowRunTriggerType | None = None,workflow_schedule_id: str | None = None,+ retried_from_workflow_run_id: str | None = None,+ fallback_attempt: int | None = None,ignore_inherited_workflow_system_prompt: bool = False,copilot_session_id: str | None = None,resolved_workflow_id: str | None = None,@@ -1478,6 +1558,8 @@ class WorkflowService:workflow_run_id=workflow_run_id,trigger_type=resolved_trigger_type,workflow_schedule_id=workflow_schedule_id,+ retried_from_workflow_run_id=retried_from_workflow_run_id,+ fallback_attempt=fallback_attempt,ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,copilot_session_id=resolved_copilot_session_id,)@@ -4542,21 +4624,29 @@ class WorkflowService:organization_id: str | None,workflow_permanent_id: str | None,) -> str:- if not parameter.credential_ids or not workflow_run_id or not organization_id or not workflow_permanent_id:+ if not workflow_run_id or not organization_id or not workflow_permanent_id:return parameter.credential_idworkflow_run_context = app.WORKFLOW_CONTEXT_MANAGER.workflow_run_contexts.get(workflow_run_id)if workflow_run_context:return await workflow_run_context.resolve_credential_parameter_id(parameter, organization_id)- return await select_credential_for_run(- workflow_run_id=workflow_run_id,- organization_id=organization_id,- workflow_permanent_id=workflow_permanent_id,- parameter_key=parameter.key,- credential_ids=parameter.credential_ids,- selection_strategy=parameter.selection_strategy,- )+ if parameter.credential_ids:+ return await select_credential_for_run(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ workflow_permanent_id=workflow_permanent_id,+ parameter_key=parameter.key,+ credential_ids=parameter.credential_ids,+ selection_strategy=parameter.selection_strategy,+ )+ if parameter.fallback_credential_ids:+ selected = await app.DATABASE.workflow_run_credential_selections.get_selection(+ workflow_run_id=workflow_run_id,+ parameter_key=parameter.key,+ )+ return selected or parameter.credential_id+ return parameter.credential_idasync def _apply_login_block_credential_proxy_pin(self,@@ -5874,6 +5964,8 @@ class WorkflowService:workflow_run_id: str | None = None,trigger_type: WorkflowRunTriggerType | None = None,workflow_schedule_id: str | None = None,+ retried_from_workflow_run_id: str | None = None,+ fallback_attempt: int | None = None,ignore_inherited_workflow_system_prompt: bool = False,copilot_session_id: str | None = None,) -> WorkflowRun:@@ -5960,6 +6052,8 @@ class WorkflowService:workflow_run_id=workflow_run_id,trigger_type=trigger_type,workflow_schedule_id=workflow_schedule_id,+ retried_from_workflow_run_id=retried_from_workflow_run_id,+ fallback_attempt=fallback_attempt,ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,copilot_session_id=copilot_session_id,)@@ -6074,6 +6168,8 @@ class WorkflowService:workflow_run_id=workflow_run_id,trigger_type=trigger_type,workflow_schedule_id=workflow_schedule_id,+ retried_from_workflow_run_id=retried_from_workflow_run_id,+ fallback_attempt=fallback_attempt,ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,copilot_session_id=copilot_session_id,)@@ -6296,13 +6392,15 @@ class WorkflowService:failure_category_source=failure_category_source,)- return await self._update_workflow_run_status(+ workflow_run = await self._update_workflow_run_status(workflow_run_id=workflow_run_id,status=WorkflowRunStatus.failed,failure_reason=failure_reason,run_with=run_with,failure_category=failure_category,)+ self._schedule_credential_fallback_retry(workflow_run)+ return workflow_runasync def mark_workflow_run_as_running(self, workflow_run_id: str, run_with: str | None = None) -> WorkflowRun:# Conditional UPDATE refuses to resurrect a finalized wr — prevents the@@ -6382,13 +6480,15 @@ class WorkflowService:failure_category_source=failure_category_source,)- return await self._update_workflow_run_status(+ workflow_run = await self._update_workflow_run_status(workflow_run_id=workflow_run_id,status=WorkflowRunStatus.terminated,failure_reason=failure_reason,run_with=run_with,failure_category=failure_category,)+ self._schedule_credential_fallback_retry(workflow_run)+ return workflow_runasync def mark_workflow_run_as_canceled(self, workflow_run_id: str) -> WorkflowRun:"""Cancel a workflow run, rejecting the transition if the run has already@@ -6998,6 +7098,7 @@ class WorkflowService:output_parameter_tuples,(recording_urls, recording_archived),(downloaded_files, downloaded_file_urls),+ retried_by_workflow_run_id,) = await asyncio.gather(self.get_recent_workflow_screenshot_urls(workflow_run_id=workflow_run_id,@@ -7009,6 +7110,10 @@ class WorkflowService:),self._fetch_recording_urls(workflow_run, task_v2, organization_id),self._fetch_downloaded_files(workflow_run, task_v2),+ app.DATABASE.workflow_runs.get_workflow_run_retried_by(+ workflow_run_id=workflow_run_id,+ organization_id=workflow_run.organization_id,+ ),)screenshot_urls: list[str] | None = screenshot_urls_raw or None# Preserve legacy singular contract: last element is the newest.@@ -7069,6 +7174,8 @@ class WorkflowService:status=workflow_run.status,failure_reason=workflow_run.failure_reason,failure_category=workflow_run.failure_category,+ retried_from_workflow_run_id=workflow_run.retried_from_workflow_run_id,+ retried_by_workflow_run_id=retried_by_workflow_run_id,proxy_location=workflow_run.proxy_location,webhook_callback_url=workflow_run.webhook_callback_url,webhook_failure_reason=workflow_run.webhook_failure_reason,diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.pyindex 13effc94d..8aa9cd742 100644--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py@@ -172,6 +172,8 @@ def convert_workflow_definition(credential_id=parameter.credential_id,credential_ids=parameter.credential_ids,selection_strategy=parameter.selection_strategy,+ fallback_credential_ids=parameter.fallback_credential_ids,+ fallback_trigger=parameter.fallback_trigger,created_at=now,modified_at=now,)diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex 093f94123..f5ddf6fa4 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -617,6 +617,8 @@ class CredentialParameterYAML(ParameterYAML):credential_id: strcredential_ids: list[str] | None = Noneselection_strategy: str | None = None+ fallback_credential_ids: list[str] | None = None+ fallback_trigger: str | None = Noneclass BitwardenSensitiveInformationParameterYAML(ParameterYAML):diff --git a/skyvern/services/workflow_service.py b/skyvern/services/workflow_service.pyindex 551cc1f73..047c2de7b 100644--- a/skyvern/services/workflow_service.py+++ b/skyvern/services/workflow_service.py@@ -16,6 +16,30 @@ from skyvern.schemas.runs import RunStatus, RunType, WorkflowRunRequest, WorkfloLOG = structlog.get_logger(__name__)+def workflow_request_body_from_existing_run(+ workflow_run: WorkflowRun,+ parameters: dict[str, t.Any] | None = None,+ run_metadata: dict[str, str] | None = None,+) -> WorkflowRequestBody:+ return WorkflowRequestBody(+ data=parameters,+ proxy_location=workflow_run.proxy_location,+ webhook_callback_url=workflow_run.webhook_callback_url,+ totp_verification_url=workflow_run.totp_verification_url,+ totp_identifier=workflow_run.totp_identifier,+ browser_session_id=workflow_run.browser_session_id,+ browser_profile_id=workflow_run.browser_profile_id,+ max_screenshot_scrolls=workflow_run.max_screenshot_scrolls,+ max_elapsed_time_minutes=workflow_run.max_elapsed_time_minutes,+ extra_http_headers=workflow_run.extra_http_headers,+ cdp_connect_headers=workflow_run.cdp_connect_headers,+ browser_address=workflow_run.browser_address,+ run_with=workflow_run.run_with,+ ai_fallback=workflow_run.ai_fallback,+ run_metadata=run_metadata,+ )++async def prepare_workflow(workflow_id: str,organization: Organization,@@ -30,6 +54,8 @@ async def prepare_workflow(trigger_type: WorkflowRunTriggerType | None = None,workflow_schedule_id: str | None = None,workflow_run_id: str | None = None,+ retried_from_workflow_run_id: str | None = None,+ fallback_attempt: int | None = None,ignore_inherited_workflow_system_prompt: bool = False,copilot_session_id: str | None = None,resolved_workflow_id: str | None = None,@@ -59,6 +85,8 @@ async def prepare_workflow(parent_workflow_run_id=parent_workflow_run_id,trigger_type=trigger_type,workflow_schedule_id=workflow_schedule_id,+ retried_from_workflow_run_id=retried_from_workflow_run_id,+ fallback_attempt=fallback_attempt,ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,copilot_session_id=copilot_session_id,resolved_workflow_id=resolved_workflow_id,@@ -110,6 +138,8 @@ async def run_workflow(parent_workflow_run_id: str | None = None,trigger_type: WorkflowRunTriggerType | None = None,workflow_schedule_id: str | None = None,+ retried_from_workflow_run_id: str | None = None,+ fallback_attempt: int | None = None,ignore_inherited_workflow_system_prompt: bool = False,tag_write_context: TagWriteContext | None = None,) -> WorkflowRun:@@ -124,6 +154,8 @@ async def run_workflow(parent_workflow_run_id=parent_workflow_run_id,trigger_type=trigger_type,workflow_schedule_id=workflow_schedule_id,+ retried_from_workflow_run_id=retried_from_workflow_run_id,+ fallback_attempt=fallback_attempt,ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,tag_write_context=tag_write_context,)diff --git a/tests/unit/test_credential_fallback.py b/tests/unit/test_credential_fallback.pynew file mode 100644index 000000000..0cc180389--- /dev/null+++ b/tests/unit/test_credential_fallback.py@@ -0,0 +1,767 @@+from __future__ import annotations++from datetime import datetime, timezone+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock, patch++import pytest+from sqlalchemy.exc import IntegrityError++from skyvern.exceptions import InvalidCredentialId, SkyvernHTTPException+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext+from skyvern.forge.sdk.workflow.credential_fallback import (+ ANY_FAILURE,+ CREDENTIAL_FAILURES,+ _trigger_matches,+ maybe_start_credential_fallback_retry,+)+from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter+from skyvern.forge.sdk.workflow.models.workflow import WorkflowRun, WorkflowRunStatus+from skyvern.forge.sdk.workflow.service import WorkflowService+from skyvern.forge.sdk.workflow.workflow_definition_converter import convert_workflow_definition+from skyvern.schemas.workflows import CredentialParameterYAML, WorkflowDefinitionYAML+++def _credential_parameter(+ *,+ key: str = "login_cred",+ credential_id: str = "cred_primary",+ credential_ids: list[str] | None = None,+ fallback_credential_ids: list[str] | None = None,+ fallback_trigger: str | None = None,+) -> CredentialParameter:+ now = datetime.now(timezone.utc)+ return CredentialParameter(+ key=key,+ credential_parameter_id=f"cp_{key}",+ workflow_id="wf_test",+ credential_id=credential_id,+ credential_ids=credential_ids,+ fallback_credential_ids=fallback_credential_ids,+ fallback_trigger=fallback_trigger,+ created_at=now,+ modified_at=now,+ )+++def _workflow_run(+ *,+ workflow_run_id: str = "wr_failed",+ status: WorkflowRunStatus = WorkflowRunStatus.failed,+ failure_category: list[dict] | None = None,+ failure_reason: str | None = "login failed",+ fallback_attempt: int | None = None,+ parent_workflow_run_id: str | None = None,+ debug_session_id: str | None = None,+ retried_from_workflow_run_id: str | None = None,+ browser_session_id: str | None = None,+ browser_profile_id: str | None = None,+ extra_http_headers: dict[str, str] | None = None,+ workflow_schedule_id: str | None = None,+) -> WorkflowRun:+ now = datetime.now(timezone.utc)+ return WorkflowRun(+ workflow_run_id=workflow_run_id,+ workflow_id="wf_test",+ workflow_permanent_id="wpid_test",+ organization_id="org_test",+ status=status,+ failure_reason=failure_reason,+ failure_category=failure_category,+ fallback_attempt=fallback_attempt,+ parent_workflow_run_id=parent_workflow_run_id,+ debug_session_id=debug_session_id,+ retried_from_workflow_run_id=retried_from_workflow_run_id,+ browser_session_id=browser_session_id,+ browser_profile_id=browser_profile_id,+ extra_http_headers=extra_http_headers,+ workflow_schedule_id=workflow_schedule_id,+ trigger_type=WorkflowRunTriggerType.api,+ created_at=now,+ modified_at=now,+ )+++def _workflow(parameters: list[CredentialParameter]) -> SimpleNamespace:+ return SimpleNamespace(+ workflow_id="wf_test",+ workflow_permanent_id="wpid_test",+ version=7,+ title="Workflow",+ workflow_definition=SimpleNamespace(parameters=parameters),+ )+++async def _validate_parameter(parameter: CredentialParameter, existing_ids: list[str] | None = None) -> None:+ service = WorkflowService()+ organization = SimpleNamespace(organization_id="org_test")+ existing = [SimpleNamespace(credential_id=credential_id) for credential_id in existing_ids or []]+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:+ mock_app.DATABASE.credentials.get_credentials_by_ids = AsyncMock(return_value=existing)+ await service._validate_and_normalize_credential_rotation_parameters([parameter], organization)+++async def test_workflow_save_validation_dedupes_fallback_credentials_preserving_order() -> None:+ parameter = _credential_parameter(+ fallback_credential_ids=["cred_b", "cred_c", "cred_b", "cred_d", "cred_c"],+ )++ await _validate_parameter(parameter, ["cred_b", "cred_c", "cred_d"])++ assert parameter.fallback_credential_ids == ["cred_b", "cred_c", "cred_d"]+ assert parameter.credential_id == "cred_primary"+++async def test_workflow_save_validation_rejects_unknown_fallback_credential_id() -> None:+ parameter = _credential_parameter(fallback_credential_ids=["cred_missing"])++ with pytest.raises(InvalidCredentialId):+ await _validate_parameter(parameter, [])+++async def test_workflow_save_validation_rejects_bad_fallback_trigger() -> None:+ parameter = _credential_parameter(fallback_credential_ids=["cred_b"], fallback_trigger="later")++ with pytest.raises(SkyvernHTTPException, match="fallback_trigger"):+ await _validate_parameter(parameter, ["cred_b"])+++async def test_workflow_save_validation_rejects_trigger_without_fallback_list() -> None:+ parameter = _credential_parameter(fallback_trigger=ANY_FAILURE)++ with pytest.raises(SkyvernHTTPException, match="requires fallback_credential_ids"):+ await _validate_parameter(parameter)+++async def test_workflow_save_validation_normalizes_empty_fallback_list_to_none() -> None:+ parameter = _credential_parameter(fallback_credential_ids=[])++ await _validate_parameter(parameter)++ assert parameter.fallback_credential_ids is None+++async def test_workflow_save_validation_drops_primary_credential_from_fallbacks() -> None:+ parameter = _credential_parameter(fallback_credential_ids=["cred_primary", "cred_b"])++ await _validate_parameter(parameter, ["cred_b"])++ assert parameter.fallback_credential_ids == ["cred_b"]+++async def test_workflow_save_validation_rejects_trigger_when_fallbacks_collapse_to_primary() -> None:+ parameter = _credential_parameter(fallback_credential_ids=["cred_primary"], fallback_trigger=ANY_FAILURE)++ with pytest.raises(SkyvernHTTPException, match="requires fallback_credential_ids"):+ await _validate_parameter(parameter)+++async def test_workflow_save_validation_rejects_rotation_with_fallback_credentials() -> None:+ parameter = _credential_parameter(+ credential_ids=["cred_primary", "cred_alt"],+ fallback_credential_ids=["cred_primary", "cred_b"],+ )++ with pytest.raises(SkyvernHTTPException) as exc_info:+ await _validate_parameter(parameter, ["cred_primary", "cred_alt", "cred_b"])++ assert exc_info.value.status_code == 400+ assert exc_info.value.message == (+ "credential parameter login_cred cannot combine credential_ids rotation with fallback_credential_ids; "+ "configure one or the other."+ )+++async def test_workflow_save_validation_allows_rotation_without_fallback_credentials() -> None:+ parameter = _credential_parameter(credential_ids=["cred_primary", "cred_alt"])++ await _validate_parameter(parameter, ["cred_primary", "cred_alt"])++ assert parameter.credential_ids == ["cred_primary", "cred_alt"]+ assert parameter.credential_id == "cred_primary"+++async def test_workflow_save_validation_allows_fallback_credentials_without_rotation() -> None:+ parameter = _credential_parameter(fallback_credential_ids=["cred_b", "cred_c"])++ await _validate_parameter(parameter, ["cred_b", "cred_c"])++ assert parameter.credential_ids is None+ assert parameter.fallback_credential_ids == ["cred_b", "cred_c"]++[email protected]("category", ["AUTH_FAILURE", "CREDENTIAL_ERROR"])+def test_credential_failure_trigger_matches_credential_categories(category: str) -> None:+ workflow_run = _workflow_run(failure_category=[{"category": category}])++ assert _trigger_matches(workflow_run, CREDENTIAL_FAILURES)+++def test_credential_failure_trigger_ignores_noncredential_categories() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "ELEMENT_NOT_FOUND"}])++ assert not _trigger_matches(workflow_run, CREDENTIAL_FAILURES)+++def test_credential_failure_trigger_falls_back_to_failure_reason_classification() -> None:+ workflow_run = _workflow_run(failure_category=None, failure_reason="The password was incorrect during login")++ assert _trigger_matches(workflow_run, CREDENTIAL_FAILURES)++[email protected]("status", [WorkflowRunStatus.failed, WorkflowRunStatus.terminated])+def test_any_failure_trigger_matches_failed_and_terminated_runs(status: WorkflowRunStatus) -> None:+ workflow_run = _workflow_run(status=status, failure_category=[{"category": "ELEMENT_NOT_FOUND"}])++ assert _trigger_matches(workflow_run, ANY_FAILURE)+++PROXY_SESSION_MARKER_HEADER = "x-proxy-session-marker"+++def _strip_marker_header(extra_http_headers: dict[str, str] | None) -> dict[str, str] | None:+ if not extra_http_headers:+ return extra_http_headers+ return {key: value for key, value in extra_http_headers.items() if key != PROXY_SESSION_MARKER_HEADER} or None+++async def _run_fallback_retry(+ *,+ workflow_run: WorkflowRun,+ parameters: list[CredentialParameter],+ existing_retry_run_id: str | None = None,+ retried_by_results: list[str | None] | None = None,+ prior_selections: dict[str, str] | None = None,+ block_scoped: bool = False,+ missing_credential_ids: set[str] | None = None,+ run_workflow_error: Exception | None = None,+ flag_enabled: bool = True,+ flag_error: Exception | None = None,+) -> tuple[str | None, AsyncMock, MagicMock]:+ captured: dict[str, object] = {}++ async def fake_run_workflow(**kwargs: object) -> WorkflowRun:+ captured["context_at_call"] = skyvern_context.current()+ if run_workflow_error is not None:+ raise run_workflow_error+ return _workflow_run(+ workflow_run_id="wr_retry",+ status=WorkflowRunStatus.created,+ failure_category=None,+ failure_reason=None,+ retried_from_workflow_run_id=kwargs["retried_from_workflow_run_id"], # type: ignore[arg-type]+ fallback_attempt=kwargs["fallback_attempt"], # type: ignore[arg-type]+ )++ run_workflow_mock = AsyncMock(side_effect=fake_run_workflow)+ run_workflow_mock.captured = captured+ with (+ patch("skyvern.forge.sdk.workflow.credential_fallback.app") as mock_app,+ patch("skyvern.services.workflow_service.run_workflow", run_workflow_mock),+ ):+ mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(+ side_effect=flag_error, return_value=flag_enabled+ )+ mock_app.AGENT_FUNCTION.is_block_scoped_workflow_run = AsyncMock(return_value=block_scoped)+ mock_app.AGENT_FUNCTION.strip_proxy_session_extra_http_headers = _strip_marker_header+ mock_app.DATABASE.debug.has_block_run_for_workflow_run = AsyncMock(return_value=False)+ mock_app.WORKFLOW_SERVICE.get_workflow = AsyncMock(return_value=_workflow(parameters))+ if retried_by_results is not None:+ mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(side_effect=retried_by_results)+ else:+ mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by = AsyncMock(return_value=existing_retry_run_id)+ mock_app.DATABASE.workflow_run_credential_selections.get_selections_for_run = AsyncMock(+ return_value=prior_selections or {}+ )++ async def fake_get_credentials_by_ids(credential_ids: list[str], organization_id: str) -> list[SimpleNamespace]:+ return [+ SimpleNamespace(credential_id=credential_id)+ for credential_id in credential_ids+ if credential_id not in (missing_credential_ids or set())+ ]++ mock_app.DATABASE.credentials.get_credentials_by_ids = AsyncMock(side_effect=fake_get_credentials_by_ids)+ mock_app.DATABASE.workflow_runs.get_workflow_run_parameters = AsyncMock(+ return_value=[+ (+ SimpleNamespace(key="account_id"),+ SimpleNamespace(value="acct_1"),+ )+ ]+ )+ mock_app.DATABASE.organizations.get_organization = AsyncMock(+ return_value=SimpleNamespace(organization_id="org_test")+ )+ result = await maybe_start_credential_fallback_retry(workflow_run, "org_test")+ return result, run_workflow_mock, mock_app+++async def test_fallback_attempt_one_uses_first_fallback_and_preserves_other_pins() -> None:+ workflow_run = _workflow_run(+ failure_category=[{"category": "AUTH_FAILURE"}],+ browser_session_id="pbs_live",+ browser_profile_id="bprof_old",+ workflow_schedule_id="ws_1",+ )+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])+ backup = _credential_parameter(+ key="backup_cred",+ credential_id="cred_backup_primary",+ credential_ids=["cred_backup_primary", "cred_backup_alt"],+ )++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login, backup],+ prior_selections={"backup_cred": "cred_backup_alt", "login_cred": "cred_primary"},+ )++ assert result == "wr_retry"+ kwargs = run_workflow_mock.await_args.kwargs+ assert kwargs["retried_from_workflow_run_id"] == "wr_failed"+ assert kwargs["fallback_attempt"] == 1+ assert kwargs["version"] == 7+ assert kwargs["trigger_type"] == WorkflowRunTriggerType.api+ assert kwargs["workflow_schedule_id"] == "ws_1"+ assert kwargs["workflow_request"].browser_session_id is None+ assert kwargs["workflow_request"].browser_profile_id is None+ assert kwargs["workflow_request"].data == {+ "account_id": "acct_1",+ "login_cred": "cred_fb1",+ "backup_cred": "cred_backup_alt",+ }+++async def test_fallback_attempt_two_advances_from_parameter_prior_selection() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], fallback_attempt=1)+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ prior_selections={"login_cred": "cred_fb1"},+ )++ assert result == "wr_retry"+ assert run_workflow_mock.await_args.kwargs["fallback_attempt"] == 2+ assert run_workflow_mock.await_args.kwargs["workflow_request"].data["login_cred"] == "cred_fb2"+++async def test_mixed_triggers_advance_each_parameter_from_its_own_prior_selection() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], fallback_attempt=1)+ any_failure_param = _credential_parameter(+ key="login_cred",+ fallback_credential_ids=["cred_a_fb1", "cred_a_fb2"],+ fallback_trigger=ANY_FAILURE,+ )+ credential_failure_param = _credential_parameter(+ key="backup_cred",+ credential_id="cred_backup_primary",+ fallback_credential_ids=["cred_b_fb1", "cred_b_fb2"],+ fallback_trigger=CREDENTIAL_FAILURES,+ )++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[any_failure_param, credential_failure_param],+ prior_selections={"login_cred": "cred_a_fb1"},+ )++ assert result == "wr_retry"+ kwargs = run_workflow_mock.await_args.kwargs+ assert kwargs["fallback_attempt"] == 2+ assert kwargs["workflow_request"].data["login_cred"] == "cred_a_fb2"+ assert kwargs["workflow_request"].data["backup_cred"] == "cred_b_fb1"+++async def test_fallback_exhaustion_creates_no_retry_run() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}], fallback_attempt=2)+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ prior_selections={"login_cred": "cred_fb2"},+ )++ assert result is None+ run_workflow_mock.assert_not_awaited()+++async def test_flag_disabled_creates_no_retry_run_and_reads_nothing() -> None:+ """Default-off gate: a run that would otherwise retry must do nothing at all.++ No retry run means no credits consumed and no webhook re-fired.+ """+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])++ result, run_workflow_mock, mock_app = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ flag_enabled=False,+ )++ assert result is None+ run_workflow_mock.assert_not_awaited()+ mock_app.WORKFLOW_SERVICE.get_workflow.assert_not_called()+ mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by.assert_not_called()+++async def test_flag_evaluation_failure_fails_closed() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ flag_error=RuntimeError("posthog is down"),+ )++ assert result is None+ run_workflow_mock.assert_not_awaited()+++async def test_deleted_fallback_credential_is_skipped_in_favor_of_next() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ missing_credential_ids={"cred_fb1"},+ )++ assert result == "wr_retry"+ kwargs = run_workflow_mock.await_args.kwargs+ assert kwargs["fallback_attempt"] == 1+ assert kwargs["workflow_request"].data["login_cred"] == "cred_fb2"+++async def test_all_fallback_credentials_deleted_does_not_advance() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(fallback_credential_ids=["cred_fb1", "cred_fb2"])++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ missing_credential_ids={"cred_fb1", "cred_fb2"},+ )++ assert result is None+ run_workflow_mock.assert_not_awaited()+++async def test_rotation_selected_prior_in_fallback_list_does_not_fake_progress() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(+ credential_id="cred_a",+ credential_ids=["cred_a", "cred_b"],+ fallback_credential_ids=["cred_c", "cred_b"],+ )++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ prior_selections={"login_cred": "cred_b"},+ )++ assert result == "wr_retry"+ kwargs = run_workflow_mock.await_args.kwargs+ assert kwargs["fallback_attempt"] == 1+ assert kwargs["workflow_request"].data["login_cred"] == "cred_c"+++async def test_retry_request_strips_credential_proxy_session_headers() -> None:+ workflow_run = _workflow_run(+ failure_category=[{"category": "AUTH_FAILURE"}],+ extra_http_headers={PROXY_SESSION_MARKER_HEADER: "proxy_sess_1", "x-custom": "keep"},+ )+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])++ result, run_workflow_mock, _ = await _run_fallback_retry(workflow_run=workflow_run, parameters=[login])++ assert result == "wr_retry"+ assert run_workflow_mock.await_args.kwargs["workflow_request"].extra_http_headers == {"x-custom": "keep"}+++async def test_retry_run_creation_does_not_inherit_ambient_skyvern_context() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])+ ambient = SkyvernContext(run_id="wr_failed", workflow_run_id="wr_failed", root_workflow_run_id="wr_failed")+ skyvern_context.set(ambient)+ try:+ result, run_workflow_mock, _ = await _run_fallback_retry(workflow_run=workflow_run, parameters=[login])+ assert skyvern_context.current() is ambient+ finally:+ skyvern_context.reset()++ assert result == "wr_retry"+ context_at_call = run_workflow_mock.captured["context_at_call"]+ assert context_at_call is None or (context_at_call.run_id is None and context_at_call.root_workflow_run_id is None)+++async def test_duplicate_retry_integrity_error_recovers_existing_retry_run() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])+ duplicate_error = IntegrityError(+ "INSERT INTO workflow_runs",+ None,+ Exception('duplicate key value violates unique constraint "ix_workflow_runs_retried_from_workflow_run_id"'),+ )++ result, _, mock_app = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ retried_by_results=[None, "wr_existing"],+ run_workflow_error=duplicate_error,+ )++ assert result == "wr_existing"+ assert mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by.await_count == 2+++async def test_unrelated_integrity_error_is_not_treated_as_duplicate_retry() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])+ unrelated_error = IntegrityError(+ "INSERT INTO workflow_runs",+ None,+ Exception('null value in column "organization_id" violates not-null constraint'),+ )++ result, _, mock_app = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ retried_by_results=[None, "wr_existing"],+ run_workflow_error=unrelated_error,+ )++ assert result is None+ assert mock_app.DATABASE.workflow_runs.get_workflow_run_retried_by.await_count == 1+++async def test_fallback_retry_idempotency_returns_existing_retry_run() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ existing_retry_run_id="wr_existing",+ )++ assert result == "wr_existing"+ run_workflow_mock.assert_not_awaited()+++ ("workflow_run", "block_scoped"),+ [+ (_workflow_run(parent_workflow_run_id="wr_parent"), False),+ (_workflow_run(debug_session_id="debug_1"), False),+ (_workflow_run(), True),+ ],+)+async def test_fallback_retry_guards_skip_nested_debug_and_block_scoped_runs(+ workflow_run: WorkflowRun,+ block_scoped: bool,+) -> None:+ login = _credential_parameter(fallback_credential_ids=["cred_fb1"])++ result, run_workflow_mock, _ = await _run_fallback_retry(+ workflow_run=workflow_run,+ parameters=[login],+ block_scoped=block_scoped,+ )++ assert result is None+ run_workflow_mock.assert_not_awaited()+++def test_credential_override_accepts_fallback_pool_and_rejects_non_pool_id() -> None:+ service = WorkflowService()+ workflow = _workflow([_credential_parameter(fallback_credential_ids=["cred_fb1"])])++ overrides = service._get_run_credential_parameter_overrides(+ workflow=workflow,+ request_data={"login_cred": "cred_fb1"},+ )++ assert overrides == {"login_cred": "cred_fb1"}+ with pytest.raises(SkyvernHTTPException, match="configured rotation or fallback credentials"):+ service._get_run_credential_parameter_overrides(+ workflow=workflow,+ request_data={"login_cred": "cred_other"},+ )+++async def test_service_resolves_fallback_only_credential_parameter_from_pinned_selection() -> None:+ service = WorkflowService()+ parameter = _credential_parameter(fallback_credential_ids=["cred_fb1"])++ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:+ mock_app.WORKFLOW_CONTEXT_MANAGER.workflow_run_contexts = {}+ mock_app.DATABASE.workflow_run_credential_selections.get_selection = AsyncMock(return_value="cred_fb1")+ selected = await service._resolve_credential_parameter_id(+ parameter=parameter,+ workflow_run_id="wr_retry",+ organization_id="org_test",+ workflow_permanent_id="wpid_test",+ )++ assert selected == "cred_fb1"+++async def test_context_resolves_fallback_only_credential_parameter_from_pinned_selection_once() -> None:+ context = WorkflowRunContext(+ workflow_title="Workflow",+ workflow_id="wf_test",+ workflow_permanent_id="wpid_test",+ workflow_run_id="wr_retry",+ aws_client=MagicMock(),+ )+ parameter = _credential_parameter(fallback_credential_ids=["cred_fb1"])++ with patch("skyvern.forge.sdk.workflow.context_manager.app") as mock_app:+ get_selection = AsyncMock(return_value="cred_fb1")+ mock_app.DATABASE.workflow_run_credential_selections.get_selection = get_selection+ first = await context.resolve_credential_parameter_id(parameter, "org_test")+ second = await context.resolve_credential_parameter_id(parameter, "org_test")++ assert first == "cred_fb1"+ assert second == "cred_fb1"+ get_selection.assert_awaited_once_with(workflow_run_id="wr_retry", parameter_key="login_cred")+++ ("pinned_selection", "expected"),+ [+ (None, "cred_runtime"),+ ("cred_fb1", "cred_fb1"),+ ],+)+async def test_context_fallback_only_parameter_applies_credential_id_indirection(+ pinned_selection: str | None,+ expected: str,+) -> None:+ context = WorkflowRunContext(+ workflow_title="Workflow",+ workflow_id="wf_test",+ workflow_permanent_id="wpid_test",+ workflow_run_id="wr_retry",+ aws_client=MagicMock(),+ )+ context.parameters["credential_source"] = MagicMock()+ context.values["credential_source"] = "cred_runtime"+ parameter = _credential_parameter(+ credential_id="credential_source",+ fallback_credential_ids=["cred_fb1"],+ )++ with patch("skyvern.forge.sdk.workflow.context_manager.app") as mock_app:+ mock_app.DATABASE.workflow_run_credential_selections.get_selection = AsyncMock(return_value=pinned_selection)+ resolved = await context.resolve_credential_parameter_id(parameter, "org_test")++ assert resolved == expected+++def test_yaml_to_credential_parameter_round_trip_preserves_fallback_fields() -> None:+ yaml_definition = WorkflowDefinitionYAML(+ parameters=[+ CredentialParameterYAML(+ key="login_cred",+ credential_id="cred_primary",+ fallback_credential_ids=["cred_fb1", "cred_fb2"],+ fallback_trigger=ANY_FAILURE,+ )+ ],+ blocks=[],+ )++ definition = convert_workflow_definition(yaml_definition, workflow_id="wf_test")+ parameter = definition.parameters[0]++ assert isinstance(parameter, CredentialParameter)+ assert parameter.fallback_credential_ids == ["cred_fb1", "cred_fb2"]+ assert parameter.fallback_trigger == ANY_FAILURE+++async def test_mark_workflow_run_as_failed_invokes_fallback_hook() -> None:+ service = WorkflowService()+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])+ service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]++ result = await service.mark_workflow_run_as_failed(+ workflow_run_id="wr_failed",+ failure_reason="login failed",+ failure_category=[{"category": "AUTH_FAILURE"}],+ )++ assert result == workflow_run+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)+++async def test_mark_workflow_run_as_terminated_invokes_fallback_hook() -> None:+ service = WorkflowService()+ workflow_run = _workflow_run(+ status=WorkflowRunStatus.terminated,+ failure_category=[{"category": "AUTH_FAILURE"}],+ )+ service._update_workflow_run_status = AsyncMock(return_value=workflow_run) # type: ignore[method-assign]+ service._schedule_credential_fallback_retry = MagicMock() # type: ignore[method-assign]++ result = await service.mark_workflow_run_as_terminated(+ workflow_run_id="wr_failed",+ failure_reason="login failed",+ failure_category=[{"category": "AUTH_FAILURE"}],+ )++ assert result == workflow_run+ service._schedule_credential_fallback_retry.assert_called_once_with(workflow_run)+++async def test_fallback_hook_exception_does_not_raise() -> None:+ workflow_run = _workflow_run(failure_category=[{"category": "AUTH_FAILURE"}])++ with patch(+ "skyvern.forge.sdk.workflow.service.maybe_start_credential_fallback_retry",+ AsyncMock(side_effect=RuntimeError("boom")),+ ):+ await WorkflowService._start_credential_fallback_retry_best_effort(workflow_run)diff --git a/tests/unit/test_credential_rotation.py b/tests/unit/test_credential_rotation.pyindex 7261cf0c8..4a03af6f5 100644--- a/tests/unit/test_credential_rotation.py+++ b/tests/unit/test_credential_rotation.py@@ -232,7 +232,7 @@ async def test_run_credential_override_rejects_credentials_outside_rotation_poolservice = WorkflowService()workflow = _setup_workflow_with_rotating_credential()- with pytest.raises(SkyvernHTTPException, match="configured rotation credentials"):+ with pytest.raises(SkyvernHTTPException, match="configured rotation or fallback credentials"):await service._apply_run_credential_parameter_overrides(workflow=workflow,workflow_run=_setup_workflow_run(),diff --git a/tests/unit/test_runs_v2_route.py b/tests/unit/test_runs_v2_route.pyindex ecfe16943..70be4eb7e 100644--- a/tests/unit/test_runs_v2_route.py+++ b/tests/unit/test_runs_v2_route.py@@ -167,6 +167,7 @@ async def test_retry_workflow_run_replays_original_run_parameters(monkeypatch: pbrowser_session_id="pbs_123",browser_profile_id="bprof_123",max_screenshot_scrolls=3,+ max_elapsed_time_minutes=None,extra_http_headers={"X-Test": "1"},cdp_connect_headers={"X-CDP-Auth": "secret"},browser_address="http://127.0.0.1:9222",@@ -357,6 +358,7 @@ async def test_retry_workflow_run_replays_template_runs_as_templates(monkeypatchbrowser_session_id=None,browser_profile_id=None,max_screenshot_scrolls=None,+ max_elapsed_time_minutes=None,extra_http_headers=None,cdp_connect_headers=None,browser_address=None,diff --git a/tests/unit/test_saved_session_proxy_pin.py b/tests/unit/test_saved_session_proxy_pin.pyindex a29de3b83..bb942eb0d 100644--- a/tests/unit/test_saved_session_proxy_pin.py+++ b/tests/unit/test_saved_session_proxy_pin.py@@ -781,6 +781,8 @@ async def test_create_workflow_run_non_force_path_single_create_no_update(monkeyworkflow_run_id=None,trigger_type=None,workflow_schedule_id=None,+ retried_from_workflow_run_id=None,+ fallback_attempt=None,ignore_inherited_workflow_system_prompt=False,copilot_session_id=None,)diff --git a/tests/unit/test_workflow_run_detail_template_cross_org.py b/tests/unit/test_workflow_run_detail_template_cross_org.pyindex 900c3aa24..640b5a669 100644--- a/tests/unit/test_workflow_run_detail_template_cross_org.py+++ b/tests/unit/test_workflow_run_detail_template_cross_org.py@@ -44,6 +44,7 @@ def _make_workflow_run() -> SimpleNamespace:status=WorkflowRunStatus.running,failure_reason=None,failure_category=None,+ retried_from_workflow_run_id=None,proxy_location=None,webhook_callback_url=None,webhook_failure_reason=None,@@ -90,6 +91,7 @@ async def test_template_run_detail_resolves_via_run_join(monkeypatch: pytest.Monworkflow_runs=SimpleNamespace(get_workflow_run_parameters=AsyncMock(return_value=[]),get_workflow_run_block_errors=AsyncMock(return_value=[]),+ get_workflow_run_retried_by=AsyncMock(return_value=None),),),)