Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: add user context field for credential login testing — frontend (#SKY-8074) (#5100)
Celal Zamanoğlud2b69f6d1c14
diff --git a/skyvern-frontend/src/api/types.ts b/skyvern-frontend/src/api/types.tsindex 41fb3e326..9b6448522 100644--- a/skyvern-frontend/src/api/types.ts+++ b/skyvern-frontend/src/api/types.ts@@ -539,6 +539,7 @@ export type CredentialApiResponse = {name: string;browser_profile_id?: string | null;tested_url?: string | null;+ user_context?: string | null;};export function isPasswordCredential(diff --git a/skyvern-frontend/src/routes/credentials/CredentialItem.tsx b/skyvern-frontend/src/routes/credentials/CredentialItem.tsxindex 8e091fdfa..e0e14012f 100644--- a/skyvern-frontend/src/routes/credentials/CredentialItem.tsx+++ b/skyvern-frontend/src/routes/credentials/CredentialItem.tsx@@ -20,9 +20,14 @@ import { credentialTypeToModalType } from "./useCredentialModalState";type Props = {credential: CredentialApiResponse;+ onStartBackgroundTest?: (+ credentialId: string,+ url: string,+ userContext?: string,+ ) => void;};-function CredentialItem({ credential }: Props) {+function CredentialItem({ credential, onStartBackgroundTest }: Props) {const [editModalOpen, setEditModalOpen] = useState(false);const credentialData = credential.credential;const modalType = credentialTypeToModalType(credential.credential_type);@@ -148,6 +153,7 @@ function CredentialItem({ credential }: Props) {onOpenChange={setEditModalOpen}editingCredential={credential}overrideType={modalType}+ onStartBackgroundTest={onStartBackgroundTest}/></div>);diff --git a/skyvern-frontend/src/routes/credentials/CredentialsList.tsx b/skyvern-frontend/src/routes/credentials/CredentialsList.tsxindex 517220862..45935bc53 100644--- a/skyvern-frontend/src/routes/credentials/CredentialsList.tsx+++ b/skyvern-frontend/src/routes/credentials/CredentialsList.tsx@@ -16,6 +16,11 @@ type CredentialFilter = "password" | "credit_card" | "secret";type Props = {filter?: CredentialFilter;+ onStartBackgroundTest?: (+ credentialId: string,+ url: string,+ userContext?: string,+ ) => void;};const EMPTY_MESSAGE: Record<CredentialFilter, string> = {@@ -26,7 +31,7 @@ const EMPTY_MESSAGE: Record<CredentialFilter, string> = {const PAGE_SIZE = 25;-function CredentialsList({ filter }: Props = {}) {+function CredentialsList({ filter, onStartBackgroundTest }: Props = {}) {const [page, setPage] = useState(1);const { data: credentials, isLoading } = useCredentialsQuery({page,@@ -75,6 +80,7 @@ function CredentialsList({ filter }: Props = {}) {<CredentialItemkey={credential.credential_id}credential={credential}+ onStartBackgroundTest={onStartBackgroundTest}/>))}</div>diff --git a/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx b/skyvern-frontend/src/routes/credentials/CredentialsModal.tsxindex d391101fb..c16718016 100644--- a/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx+++ b/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx@@ -40,6 +40,7 @@ import { useCredentialsQuery } from "@/routes/workflows/hooks/useCredentialsQuerimport { Checkbox } from "@/components/ui/checkbox";import { HelpTooltip } from "@/components/HelpTooltip";import { Label } from "@/components/ui/label";+import { Textarea } from "@/components/ui/textarea";import { getHostname } from "@/util/getHostname";const PASSWORD_CREDENTIAL_INITIAL_VALUES = {@@ -108,7 +109,11 @@ type Props = {/** Override the modal type (used in edit mode to set the correct form) */overrideType?: CredentialModalType;/** Called after a credential is saved with "Save browser session" checked to trigger an async test */- onStartBackgroundTest?: (credentialId: string, url: string) => void;+ onStartBackgroundTest?: (+ credentialId: string,+ url: string,+ userContext?: string,+ ) => void;};function CredentialsModal({@@ -161,6 +166,7 @@ function CredentialsModal({// Test & Save Browser Profile stateconst [testAndSave, setTestAndSave] = useState(false);const [testUrl, setTestUrl] = useState("");+ const [userContext, setUserContext] = useState("");const [testStatus, setTestStatus] = useState<"idle" | "testing" | "completed" | "failed" | "profile_failed">("idle");@@ -186,7 +192,9 @@ function CredentialsModal({const saveIntentRef = useRef<{shouldTestAfterSave: boolean;testUrl: string;- }>({ shouldTestAfterSave: false, testUrl: "" });+ userContext: string;+ name: string;+ }>({ shouldTestAfterSave: false, testUrl: "", userContext: "", name: "" });// Cleanup polling on unmountuseEffect(() => {@@ -214,7 +222,7 @@ function CredentialsModal({setTestCredentialId(null);testCredentialIdRef.current = null;}- // eslint-disable-next-line react-hooks/exhaustive-deps -- only react to login-affecting field changes, not testStatus or name+ // eslint-disable-next-line react-hooks/exhaustive-deps -- only react to login-affecting field changes, not testStatus, name, or userContext (context is cosmetic, not credential identity)}, [passwordCredentialValues.username,passwordCredentialValues.password,@@ -236,6 +244,15 @@ function CredentialsModal({if (isEditMode) {reset();const cred = editingCredential.credential;+ if (editingCredential.tested_url) {+ setTestUrl(editingCredential.tested_url);+ }+ if (editingCredential.browser_profile_id) {+ setTestAndSave(true);+ }+ if (editingCredential.user_context) {+ setUserContext(editingCredential.user_context);+ }if (isPasswordCredential(cred)) {setPasswordCredentialValues({name: editingCredential.name,@@ -299,7 +316,13 @@ function CredentialsModal({pollStartTimeRef.current = null;pollErrorCountRef.current = 0;pollCancelledRef.current = false;- saveIntentRef.current = { shouldTestAfterSave: false, testUrl: "" };+ setUserContext("");+ saveIntentRef.current = {+ shouldTestAfterSave: false,+ testUrl: "",+ userContext: "",+ name: "",+ };if (pollIntervalRef.current) {clearTimeout(pollIntervalRef.current);pollIntervalRef.current = null;@@ -445,6 +468,7 @@ function CredentialsModal({totp_type: passwordCredentialValues.totp_type,totp_identifier:passwordCredentialValues.totp_identifier.trim() || null,+ user_context: userContext.trim() || null,},);const data = response.data;@@ -485,7 +509,13 @@ function CredentialsModal({variant: "destructive",});}- }, [credentialGetter, testUrl, passwordCredentialValues, pollTestStatus]);+ }, [+ credentialGetter,+ testUrl,+ passwordCredentialValues,+ userContext,+ pollTestStatus,+ ]);const createCredentialMutation = useMutation({mutationFn: async (request: CreateCredentialRequest) => {@@ -494,16 +524,20 @@ function CredentialsModal({return response.data;},onSuccess: async (data) => {- const { shouldTestAfterSave, testUrl: capturedTestUrl } =- saveIntentRef.current;-- // If the user entered a URL, save it on the credential as metadata- if (capturedTestUrl) {+ const {+ shouldTestAfterSave,+ testUrl: capturedTestUrl,+ userContext: capturedUserContext,+ } = saveIntentRef.current;++ // Save metadata (tested_url, user_context) on the credential via PATCH+ if (capturedTestUrl || capturedUserContext) {try {const client = await getClient(credentialGetter, "sans-api-v1");await client.patch(`/credentials/${data.credential_id}`, {name: data.name,- tested_url: capturedTestUrl,+ ...(capturedTestUrl && { tested_url: capturedTestUrl }),+ user_context: capturedUserContext?.trim() || null,});} catch {// Best-effort — credential was created, URL is just metadata@@ -517,7 +551,11 @@ function CredentialsModal({setIsOpen(false);if (shouldTestAfterSave && onStartBackgroundTest) {- onStartBackgroundTest(data.credential_id, capturedTestUrl);+ onStartBackgroundTest(+ data.credential_id,+ capturedTestUrl,+ capturedUserContext || undefined,+ );toast({title: "Credential saved",description:@@ -559,17 +597,66 @@ function CredentialsModal({);return response.data;},- onSuccess: () => {+ onSuccess: async () => {+ const {+ shouldTestAfterSave,+ testUrl: capturedTestUrl,+ userContext: capturedUserContext,+ name: capturedName,+ } = saveIntentRef.current;++ // Persist metadata (tested_url, user_context) via PATCH+ if (editingCredential?.credential_id) {+ try {+ const client = await getClient(credentialGetter, "sans-api-v1");+ await client.patch(+ `/credentials/${editingCredential.credential_id}`,+ {+ name: capturedName || editingCredential.name,+ ...(capturedTestUrl && { tested_url: capturedTestUrl }),+ user_context: capturedUserContext?.trim() || null,+ },+ );+ } catch {+ toast({+ title: "Partial save",+ description:+ "Credential updated, but login instructions could not be saved. Please try editing again.",+ variant: "destructive",+ });+ }+ }+reset();setIsOpen(false);queryClient.invalidateQueries({queryKey: ["credentials"],});- toast({- title: "Credential updated",- description: "Your credential has been updated successfully",- variant: "success",- });++ if (+ shouldTestAfterSave &&+ capturedTestUrl &&+ editingCredential?.credential_id &&+ onStartBackgroundTest+ ) {+ onStartBackgroundTest(+ editingCredential.credential_id,+ capturedTestUrl,+ capturedUserContext || undefined,+ );+ toast({+ title: "Credential updated",+ description:+ "Testing login and saving browser session in the background…",+ variant: "success",+ });+ } else {+ toast({+ title: "Credential updated",+ description: "Your credential has been updated successfully",+ variant: "success",+ });+ }},onError: (error: AxiosError) => {const detail = (error.response?.data as { detail?: string })?.detail;@@ -586,16 +673,21 @@ function CredentialsModal({id,name,tested_url,+ user_context,}: {id: string;name: string;tested_url?: string;+ user_context?: string | null;}) => {const client = await getClient(credentialGetter, "sans-api-v1");- const body: Record<string, string> = { name };+ const body: Record<string, string | null> = { name };if (tested_url) {body.tested_url = tested_url;}+ if (user_context !== undefined) {+ body.user_context = user_context;+ }const response = await client.patch<CredentialApiResponse>(`/credentials/${id}`,body,@@ -691,19 +783,33 @@ function CredentialsModal({// If test passed, rename the temp credential instead of creating a new oneif (testAndSave && testStatus === "completed" && testCredentialId) {const url = testUrl.trim();+ const ctx = userContext.trim();renameCredentialMutation.mutate({id: testCredentialId,name,tested_url: url || undefined,+ user_context: ctx || null,});return;}// Capture intent before mutation — state will be reset in onSuccess+ // In edit mode, only trigger a background test if the user actually changed+ // credentials, user_context, or URL — not just because the checkbox was pre-checked.+ const hasEditModeChanges =+ !isEditMode ||+ editingGroups.values ||+ userContext.trim() !== (editingCredential?.user_context ?? "") ||+ testUrl.trim() !== (editingCredential?.tested_url ?? "");saveIntentRef.current = {shouldTestAfterSave:- testAndSave && testStatus !== "completed" && testUrl.trim() !== "",+ testAndSave &&+ testStatus !== "completed" &&+ testUrl.trim() !== "" &&+ hasEditModeChanges,testUrl: testUrl.trim(),+ userContext: userContext.trim(),+ name,};activeMutation.mutate({@@ -826,7 +932,7 @@ function CredentialsModal({values={passwordCredentialValues}onChange={setPasswordCredentialValues}url={testUrl}- onUrlChange={!isEditMode ? setTestUrl : undefined}+ onUrlChange={setTestUrl}urlRequired={testAndSave}urlDisabled={isTestInProgress}editMode={isEditMode}@@ -834,72 +940,92 @@ function CredentialsModal({onEnableEditName={handleEnableEditName}onEnableEditValues={handleEnableEditValues}afterUrl={- !isEditMode ? (- <div className="space-y-3">- <div className="flex items-center gap-3">- <Checkbox- id="test-and-save"- checked={testAndSave}- onCheckedChange={(checked) =>- setTestAndSave(checked === true)- }- disabled={isTestInProgress}- />+ <div className="space-y-3">+ <div className="flex items-center gap-3">+ <Checkbox+ id="test-and-save"+ checked={testAndSave}+ onCheckedChange={(checked) =>+ setTestAndSave(checked === true)+ }+ disabled={isTestInProgress}+ />+ <Label+ htmlFor="test-and-save"+ className="cursor-pointer text-sm font-medium"+ >+ Save browser session for future logins+ </Label>+ <HelpTooltip content="Skyvern will log in using your credentials, verify success, and save the browser session. Future workflow runs will skip the login form entirely because the saved session is already authenticated." />+ </div>++ {testAndSave && (+ <div className="space-y-1 pl-7"><Label- htmlFor="test-and-save"- className="cursor-pointer text-sm font-medium"+ htmlFor="user-context"+ className="text-xs text-muted-foreground">- Save browser session for future logins+ Login instructions (optional)</Label>- <HelpTooltip content="Skyvern will log in using your credentials, verify success, and save the browser session. Future workflow runs will skip the login form entirely because the saved session is already authenticated." />+ {/* maxLength is intentionally lower than the backend's 1000-char limit (defense-in-depth) */}+ <Textarea+ id="user-context"+ value={userContext}+ onChange={(e) => setUserContext(e.target.value)}+ placeholder='Describe the login flow, e.g. "Click the SSO button first, then enter Google credentials"'+ disabled={isTestInProgress}+ className="min-h-[60px] resize-y"+ rows={2}+ maxLength={500}+ /></div>+ )}- {isTestInProgress && (- <div className="flex items-center gap-2 pl-7 text-sm text-muted-foreground">- <ReloadIcon className="size-4 animate-spin" />- <span>{TEST_STATUS_MESSAGES[testMessageIndex]}</span>+ {isTestInProgress && (+ <div className="flex items-center gap-2 pl-7 text-sm text-muted-foreground">+ <ReloadIcon className="size-4 animate-spin" />+ <span>{TEST_STATUS_MESSAGES[testMessageIndex]}</span>+ </div>+ )}+ {testStatus === "completed" && (+ <div className="flex items-center gap-2 pl-7 text-sm text-green-400">+ <CheckCircledIcon className="size-4" />+ <span>+ {`Login test passed — saved browser session available for workflows using ${getHostname(testUrl) ?? testUrl}`}+ </span>+ </div>+ )}+ {testStatus === "profile_failed" && (+ <div className="space-y-1 pl-7">+ <div className="flex items-center gap-2 text-sm text-destructive">+ <CrossCircledIcon className="size-4" />+ <span>Browser profile was not saved</span></div>- )}- {testStatus === "completed" && (- <div className="flex items-center gap-2 pl-7 text-sm text-green-400">- <CheckCircledIcon className="size-4" />+ {testFailureReason && (+ <p className="text-xs text-destructive/70">+ {testFailureReason}+ </p>+ )}+ </div>+ )}+ {testStatus === "failed" && (+ <div className="space-y-1 pl-7">+ <div className="flex items-center gap-2 text-sm text-destructive">+ <CrossCircledIcon className="size-4" /><span>- {`Login test passed — saved browser session available for workflows using ${getHostname(testUrl) ?? testUrl}`}+ {testUrl+ ? `Unable to save browser session for ${getHostname(testUrl) ?? testUrl}`+ : "Unable to save browser session"}</span></div>- )}- {testStatus === "profile_failed" && (- <div className="space-y-1 pl-7">- <div className="flex items-center gap-2 text-sm text-destructive">- <CrossCircledIcon className="size-4" />- <span>Browser profile was not saved</span>- </div>- {testFailureReason && (- <p className="text-xs text-destructive/70">- {testFailureReason}- </p>- )}- </div>- )}- {testStatus === "failed" && (- <div className="space-y-1 pl-7">- <div className="flex items-center gap-2 text-sm text-destructive">- <CrossCircledIcon className="size-4" />- <span>- {testUrl- ? `Unable to save browser session for ${getHostname(testUrl) ?? testUrl}`- : "Unable to save browser session"}- </span>- </div>- {testFailureReason && (- <p className="text-xs text-destructive/70">- {testFailureReason}- </p>- )}- </div>- )}- </div>- ) : undefined+ {testFailureReason && (+ <p className="text-xs text-destructive/70">+ {testFailureReason}+ </p>+ )}+ </div>+ )}+ </div>}/>);@@ -1001,7 +1127,10 @@ function CredentialsModal({}, [credentialGetter]);// Whether the Test button should be shown- const showTestButton = testAndSave && type === CredentialModalTypes.PASSWORD;+ const showTestButton =+ testAndSave &&+ type === CredentialModalTypes.PASSWORD &&+ (!isEditMode || editingGroups.values);// Whether the Test button should be enabledconst canTest =diff --git a/skyvern-frontend/src/routes/credentials/CredentialsPage.tsx b/skyvern-frontend/src/routes/credentials/CredentialsPage.tsxindex acc183129..b165422c0 100644--- a/skyvern-frontend/src/routes/credentials/CredentialsPage.tsx+++ b/skyvern-frontend/src/routes/credentials/CredentialsPage.tsx@@ -112,15 +112,24 @@ function CredentialsPage() {</TabsList><TabsContent value="passwords" className="space-y-4">- <CredentialsList filter="password" />+ <CredentialsList+ filter="password"+ onStartBackgroundTest={startBackgroundTest}+ /></TabsContent><TabsContent value="creditCards" className="space-y-4">- <CredentialsList filter="credit_card" />+ <CredentialsList+ filter="credit_card"+ onStartBackgroundTest={startBackgroundTest}+ /></TabsContent><TabsContent value="secrets" className="space-y-4">- <CredentialsList filter="secret" />+ <CredentialsList+ filter="secret"+ onStartBackgroundTest={startBackgroundTest}+ /></TabsContent><TabsContent value="twoFactor" className="space-y-4">diff --git a/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.ts b/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.tsindex ad8edf870..2db413152 100644--- a/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.ts+++ b/skyvern-frontend/src/routes/credentials/useBackgroundCredentialTest.ts@@ -26,7 +26,7 @@ type ActiveTest = {* Hook that manages background credential browser-profile tests.** After a credential is saved with "Save browser session" checked,- * call `startBackgroundTest(credentialId, url)` to kick off an async test.+ * call `startBackgroundTest(credentialId, url, userContext?)` to kick off an async test.* The hook polls the backend, shows toast notifications on completion/failure,* and invalidates the credentials query so the list updates.*@@ -155,7 +155,7 @@ function useBackgroundCredentialTest() {}, [credentialGetter, queryClient, cleanup]);const startBackgroundTest = useCallback(- async (credentialId: string, url: string) => {+ async (credentialId: string, url: string, userContext?: string) => {// Clean up any previous testcleanup();@@ -166,6 +166,7 @@ function useBackgroundCredentialTest() {{url,save_browser_profile: true,+ user_context: userContext?.trim() || null,},);const data = response.data;