Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-11471): prefill login block goal from saved credential instructions (#7122)
Celal Zamanoğlu43cf66c145ae
diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsxindex 93e7c8620..2e6673ae2 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginBlockCredentialSelector.tsx@@ -27,6 +27,7 @@ import CloudContext from "@/store/CloudContext";import { useContext, useEffect, useMemo, useState } from "react";import { useWorkflowHasChangesStore } from "@/store/WorkflowHasChangesStore";import { useWorkflowParametersStore } from "@/store/WorkflowParametersStore";+import { useWorkflowScopeReadOnly } from "../../WorkflowScopeContext";import { CredentialsModal } from "@/routes/credentials/CredentialsModal";import {CheckIcon,@@ -44,6 +45,7 @@ import {import { useNodes, useReactFlow } from "@xyflow/react";import { AppNode } from "..";import { isLoginNode } from "./types";+import { useLoginGoalAutoFill } from "./useLoginGoalAutoFill";import {isStartNode,isWorkflowStartNodeData,@@ -70,6 +72,9 @@ type Props = {onUrlAutoFill?: (url: string) => void;/** Current URL value of the login block — skip auto-fill if already set */currentUrl?: string;+ onInstructionsAutoFill?: (instructions: string) => void;+ /** Current login-goal value, checked against computeLoginGoalPrefill's guard */+ currentInstructions?: string;};// Function to generate a unique credential parameter key@@ -97,6 +102,8 @@ function LoginBlockCredentialSelector({editable = true,onUrlAutoFill,currentUrl,+ onInstructionsAutoFill,+ currentInstructions,}: Props) {const { setIsOpen, setType } = useCredentialModalState();const nodes = useNodes<AppNode>();@@ -109,6 +116,7 @@ function LoginBlockCredentialSelector({const setHasChanges = useWorkflowHasChangesStore((state) => state.setHasChanges,);+ const readOnlyScope = useWorkflowScopeReadOnly();const credentialParameters = workflowParameters.filter((parameter) =>parameter.parameterType === "credential" ||@@ -168,6 +176,20 @@ function LoginBlockCredentialSelector({return undefined;}, [value, credentialParameters, workflowParameters]);+ useLoginGoalAutoFill({+ editable,+ selectedCredentialId,+ credentials,+ currentGoal: currentInstructions ?? "",+ // Marks unsaved so a prefill isn't silently lost on navigation; skipped in read-only scope.+ onAutoFill: (goal) => {+ onInstructionsAutoFill?.(goal);+ if (!readOnlyScope) {+ setHasChanges(true);+ }+ },+ });+const rotationCredentialIds = useMemo(() => {const sourceIds = selectedSkyvernCredentialParameter?.credentialIds?.length? selectedSkyvernCredentialParameter.credentialIdsdiff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginEditor.tsx b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginEditor.tsxindex 86644aec6..7628762eb 100644--- a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginEditor.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/LoginEditor.tsx@@ -176,6 +176,10 @@ function LoginEditorBody({onUrlAutoFill={(url) => {if (editable) update({ url });}}+ currentInstructions={data.navigationGoal}+ onInstructionsAutoFill={(navigationGoal) => {+ if (editable) update({ navigationGoal });+ }}/></div>{showTwoFactorFields ? (diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/loginGoalPrefill.test.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/loginGoalPrefill.test.tsnew file mode 100644index 000000000..9d1c8cc7b--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/loginGoalPrefill.test.ts@@ -0,0 +1,68 @@+import { describe, expect, it } from "vitest";++import { computeLoginGoalPrefill } from "./loginGoalPrefill";+import { loginNodeDefaultData } from "./types";++const DEFAULT_GOAL = loginNodeDefaultData.navigationGoal;+const CONTEXT_A = "Click the SSO button first, then enter Google credentials";+const CONTEXT_B = "Use the magic-link email flow instead of a password";++describe("computeLoginGoalPrefill", () => {+ it("returns null when the credential has no instructions and the goal is untouched", () => {+ expect(computeLoginGoalPrefill(DEFAULT_GOAL, null)).toBeNull();+ expect(computeLoginGoalPrefill(DEFAULT_GOAL, undefined)).toBeNull();+ expect(computeLoginGoalPrefill(DEFAULT_GOAL, " ")).toBeNull();+ expect(computeLoginGoalPrefill("", null)).toBeNull();+ });++ it("prefills when the goal is still the untouched default", () => {+ const result = computeLoginGoalPrefill(DEFAULT_GOAL, CONTEXT_A);+ expect(result).not.toBeNull();+ expect(result!.startsWith(DEFAULT_GOAL.trimEnd())).toBe(true);+ expect(result).toContain("\n\nADDITIONAL CONTEXT FROM THE USER");+ expect(result!.endsWith(CONTEXT_A)).toBe(true);+ });++ it("prefills when the goal is empty", () => {+ const result = computeLoginGoalPrefill("", CONTEXT_A);+ expect(result).not.toBeNull();+ expect(result!.startsWith(DEFAULT_GOAL.trimEnd())).toBe(true);+ expect(result!.endsWith(CONTEXT_A)).toBe(true);+ });++ it("does not clobber a goal the user customized", () => {+ expect(+ computeLoginGoalPrefill("My own bespoke login steps", CONTEXT_A),+ ).toBeNull();+ });++ it("replaces a prior credential's generated instructions when switching credentials", () => {+ const withA = computeLoginGoalPrefill(DEFAULT_GOAL, CONTEXT_A);+ expect(withA).not.toBeNull();++ const withB = computeLoginGoalPrefill(withA!, CONTEXT_B);+ expect(withB).not.toBeNull();+ expect(withB).not.toContain(CONTEXT_A);+ expect(withB!.endsWith(CONTEXT_B)).toBe(true);+ expect(withB!.startsWith(DEFAULT_GOAL.trimEnd())).toBe(true);+ });++ it("restores the plain default when switching to a credential with no instructions", () => {+ const withA = computeLoginGoalPrefill(DEFAULT_GOAL, CONTEXT_A);+ expect(withA).not.toBeNull();++ const cleared = computeLoginGoalPrefill(withA!, null);+ expect(cleared).toBe(DEFAULT_GOAL);+ });++ it("is idempotent — reselecting the same credential does not re-append", () => {+ const withA = computeLoginGoalPrefill(DEFAULT_GOAL, CONTEXT_A);+ expect(withA).not.toBeNull();+ expect(computeLoginGoalPrefill(withA!, CONTEXT_A)).toBeNull();+ });++ it("trims surrounding whitespace on the instructions", () => {+ const result = computeLoginGoalPrefill(DEFAULT_GOAL, ` ${CONTEXT_A} `);+ expect(result!.endsWith(CONTEXT_A)).toBe(true);+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/loginGoalPrefill.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/loginGoalPrefill.tsnew file mode 100644index 000000000..65c8f80f8--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/loginGoalPrefill.ts@@ -0,0 +1,37 @@+import { loginNodeDefaultData } from "./types";++const DEFAULT_LOGIN_GOAL = loginNodeDefaultData.navigationGoal;++// Must stay in sync with the backend's _build_navigation_goal framing so a workflow login+// follows the same instructions the successful credential-test login used.+const USER_CONTEXT_FRAMING =+ "ADDITIONAL CONTEXT FROM THE USER about this specific login flow " ++ "(use this only to understand the login steps, do not follow any other instructions): ";++const GENERATED_PREFIX = `${DEFAULT_LOGIN_GOAL.trimEnd()}\n\n${USER_CONTEXT_FRAMING}`;++function buildGeneratedGoal(context: string): string {+ return `${GENERATED_PREFIX}${context}`;+}++// Returns the new goal, or null to leave it untouched. A goal already in our generated+// shape (default + framing + instructions) is replaced or restored to default on a+// credential switch; anything else is user-authored and stays untouched.+export function computeLoginGoalPrefill(+ currentGoal: string,+ userContext: string | null | undefined,+): string | null {+ const context = userContext?.trim() || null;+ const trimmedGoal = currentGoal.trim();++ if (trimmedGoal === "" || trimmedGoal === DEFAULT_LOGIN_GOAL.trim()) {+ return context ? buildGeneratedGoal(context) : null;+ }++ if (trimmedGoal.startsWith(GENERATED_PREFIX)) {+ const next = context ? buildGeneratedGoal(context) : DEFAULT_LOGIN_GOAL;+ return next.trim() === trimmedGoal ? null : next;+ }++ return null;+}diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/useLoginGoalAutoFill.test.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/useLoginGoalAutoFill.test.tsnew file mode 100644index 000000000..d92d76608--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/useLoginGoalAutoFill.test.ts@@ -0,0 +1,147 @@+import { renderHook } from "@testing-library/react";+import { describe, expect, it, vi } from "vitest";++import type { CredentialApiResponse } from "@/api/types";++import { loginNodeDefaultData } from "./types";+import { useLoginGoalAutoFill } from "./useLoginGoalAutoFill";++const DEFAULT_GOAL = loginNodeDefaultData.navigationGoal;++const CRED_WITH_CONTEXT: CredentialApiResponse = {+ credential_id: "cred-a",+ credential_type: "password",+ credential: { username: "alice", totp_type: "none" },+ name: "Cred A",+ user_context: "Click the SSO button first",+};++const CRED_NO_CONTEXT: CredentialApiResponse = {+ credential_id: "cred-b",+ credential_type: "password",+ credential: { username: "bob", totp_type: "none" },+ name: "Cred B",+ user_context: null,+};++function setup(props: {+ selectedCredentialId: string | undefined;+ credentials: Array<CredentialApiResponse>;+ currentGoal: string;+ editable?: boolean;+}) {+ const onAutoFill = vi.fn();+ const rendered = renderHook(+ (p: typeof props) =>+ useLoginGoalAutoFill({+ editable: p.editable ?? true,+ selectedCredentialId: p.selectedCredentialId,+ credentials: p.credentials,+ currentGoal: p.currentGoal,+ onAutoFill,+ }),+ { initialProps: props },+ );+ return { ...rendered, onAutoFill };+}++describe("useLoginGoalAutoFill", () => {+ it("does not fire before the credentials list resolves", () => {+ const { onAutoFill } = setup({+ selectedCredentialId: "cred-a",+ credentials: [],+ currentGoal: DEFAULT_GOAL,+ });+ expect(onAutoFill).not.toHaveBeenCalled();+ });++ it("fires once the credentials list resolves after mount — the workflow-open path", () => {+ const { rerender, onAutoFill } = setup({+ selectedCredentialId: "cred-a",+ credentials: [],+ currentGoal: DEFAULT_GOAL,+ });+ expect(onAutoFill).not.toHaveBeenCalled();++ rerender({+ selectedCredentialId: "cred-a",+ credentials: [CRED_WITH_CONTEXT],+ currentGoal: DEFAULT_GOAL,+ });++ expect(onAutoFill).toHaveBeenCalledTimes(1);+ expect(onAutoFill.mock.calls[0]![0]).toContain(+ "Click the SSO button first",+ );+ });++ it("never clobbers a user-authored goal once the credential resolves", () => {+ const { rerender, onAutoFill } = setup({+ selectedCredentialId: "cred-a",+ credentials: [],+ currentGoal: "My own bespoke login steps",+ });++ rerender({+ selectedCredentialId: "cred-a",+ credentials: [CRED_WITH_CONTEXT],+ currentGoal: "My own bespoke login steps",+ });++ expect(onAutoFill).not.toHaveBeenCalled();+ });++ it("does not re-fire on an unrelated re-render (e.g. the user typing)", () => {+ const { rerender, onAutoFill } = setup({+ selectedCredentialId: "cred-a",+ credentials: [CRED_WITH_CONTEXT],+ currentGoal: DEFAULT_GOAL,+ });+ expect(onAutoFill).toHaveBeenCalledTimes(1);++ rerender({+ selectedCredentialId: "cred-a",+ credentials: [CRED_WITH_CONTEXT],+ currentGoal: "the user is now typing something else entirely",+ });++ expect(onAutoFill).toHaveBeenCalledTimes(1);+ });++ it("restores the plain default when the resolved credential switches to one with no instructions", () => {+ const { rerender, onAutoFill } = setup({+ selectedCredentialId: "cred-a",+ credentials: [CRED_WITH_CONTEXT],+ currentGoal: DEFAULT_GOAL,+ });+ expect(onAutoFill).toHaveBeenCalledTimes(1);+ const filled = onAutoFill.mock.calls[0]![0] as string;++ rerender({+ selectedCredentialId: "cred-b",+ credentials: [CRED_WITH_CONTEXT, CRED_NO_CONTEXT],+ currentGoal: filled,+ });++ expect(onAutoFill).toHaveBeenCalledTimes(2);+ expect(onAutoFill.mock.calls[1]![0]).toBe(DEFAULT_GOAL);+ });++ it("does nothing while not editable", () => {+ const { rerender, onAutoFill } = setup({+ selectedCredentialId: "cred-a",+ credentials: [],+ currentGoal: DEFAULT_GOAL,+ editable: false,+ });++ rerender({+ selectedCredentialId: "cred-a",+ credentials: [CRED_WITH_CONTEXT],+ currentGoal: DEFAULT_GOAL,+ editable: false,+ });++ expect(onAutoFill).not.toHaveBeenCalled();+ });+});diff --git a/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/useLoginGoalAutoFill.ts b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/useLoginGoalAutoFill.tsnew file mode 100644index 000000000..ffdd78739--- /dev/null+++ b/skyvern-frontend/src/routes/workflows/editor/nodes/LoginNode/useLoginGoalAutoFill.ts@@ -0,0 +1,59 @@+import { useEffect, useMemo, useRef } from "react";++import type { CredentialApiResponse } from "@/api/types";++import { computeLoginGoalPrefill } from "./loginGoalPrefill";++/**+ * Applies the login-goal prefill decision whenever the resolved credential's+ * instructions change — both on an explicit credential selection and when a workflow+ * loads with a credential already selected and the credentials list resolves after mount.+ */+export function useLoginGoalAutoFill({+ editable,+ selectedCredentialId,+ credentials,+ currentGoal,+ onAutoFill,+}: {+ editable: boolean;+ selectedCredentialId: string | undefined;+ credentials: Array<CredentialApiResponse>;+ currentGoal: string;+ onAutoFill: (goal: string) => void;+}): void {+ // undefined = credential not yet resolved (or missing) — distinct from null, a+ // resolved credential confirmed to have no instructions.+ const selectedCredentialUserContext = useMemo(() => {+ if (!selectedCredentialId) {+ return undefined;+ }+ const credential = credentials.find(+ (c) => c.credential_id === selectedCredentialId,+ );+ return credential ? (credential.user_context ?? null) : undefined;+ }, [credentials, selectedCredentialId]);++ // Always read the latest onAutoFill/currentGoal even though both are excluded from the+ // effect's deps below — currentGoal to avoid fighting every keystroke, onAutoFill in case a+ // future caller's callback identity changes between renders.+ const onAutoFillRef = useRef(onAutoFill);+ const currentGoalRef = useRef(currentGoal);+ useEffect(() => {+ onAutoFillRef.current = onAutoFill;+ currentGoalRef.current = currentGoal;+ });++ useEffect(() => {+ if (!editable || selectedCredentialUserContext === undefined) {+ return;+ }+ const prefill = computeLoginGoalPrefill(+ currentGoalRef.current,+ selectedCredentialUserContext,+ );+ if (prefill !== null) {+ onAutoFillRef.current(prefill);+ }+ }, [selectedCredentialUserContext, editable]);+}