Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Show the matching credentials popup from Record Browser (#SKY-15140) (#8323)
Cindy Li1287a6d10d2c
diff --git a/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx b/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsxindex 2550dad37..664ed6015 100644--- a/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx+++ b/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx@@ -1378,3 +1378,77 @@ describe("CredentialsModal run-sequentially toggle", () => {);}, 10_000);});++describe("CredentialsModal record-browser defaults", () => {+ function renderRecordingModal(opts: {+ overrideType?: (typeof CredentialModalTypes)[keyof typeof CredentialModalTypes];+ defaultTotpType?: "authenticator" | "email";+ heading?: string;+ }) {+ const queryClient = new QueryClient({+ defaultOptions: {+ queries: { retry: false },+ mutations: { retry: false },+ },+ });+ return render(+ <QueryClientProvider client={queryClient}>+ <MemoryRouter>+ <CredentialsModal+ isOpen+ onOpenChange={vi.fn()}+ overrideType={opts.overrideType ?? CredentialModalTypes.PASSWORD}+ defaultTotpType={opts.defaultTotpType}+ heading={opts.heading}+ />+ </MemoryRouter>+ </QueryClientProvider>,+ );+ }++ it("uses the supplied heading and opens 2FA on the authenticator method", async () => {+ renderRecordingModal({+ defaultTotpType: "authenticator",+ heading: "Add Two-Factor Authentication",+ });++ expect(+ await screen.findByRole("heading", {+ name: "Add Two-Factor Authentication",+ }),+ ).toBeTruthy();+ // Every method tile renders whenever the accordion is open, so presence alone passes+ // for defaultTotpType "email" too; aria-pressed is what distinguishes the selection.+ const authenticatorMethod = await screen.findByRole("button", {+ name: /authenticator app/i,+ });+ expect(authenticatorMethod.getAttribute("aria-pressed")).toBe("true");+ const emailMethod = await screen.findByRole("button", { name: /email/i });+ expect(emailMethod.getAttribute("aria-pressed")).toBe("false");+ });++ it("opens 2FA on the email method for a magic-link prompt", async () => {+ renderRecordingModal({+ defaultTotpType: "email",+ heading: "Add Magic Link",+ });++ expect(+ await screen.findByRole("heading", { name: "Add Magic Link" }),+ ).toBeTruthy();+ const emailMethod = await screen.findByRole("button", { name: /email/i });+ expect(emailMethod.getAttribute("aria-pressed")).toBe("true");+ });++ it("renders the secret form for a secret override", async () => {+ renderRecordingModal({+ overrideType: CredentialModalTypes.SECRET,+ heading: "Add Secret",+ });++ expect(+ await screen.findByRole("heading", { name: "Add Secret" }),+ ).toBeTruthy();+ expect(screen.getByText("Secret Label (optional)")).toBeTruthy();+ });+});diff --git a/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx b/skyvern-frontend/src/routes/credentials/CredentialsModal.tsxindex 9504e9a79..c7b342cc9 100644--- a/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx+++ b/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx@@ -258,6 +258,8 @@ type Props = {// tested_url and matches later hostname-keyed asks. Create mode only; leaving// it undefined (every non-copilot caller) keeps the field empty as before.defaultTestUrl?: string;+ defaultTotpType?: PasswordCredential["totp_type"];+ heading?: string;/** Called after a credential is saved with "Save browser session" checked to trigger an async test */onStartBackgroundTest?: (credentialId: string,@@ -295,6 +297,8 @@ function CredentialsModal({editingCredential,overrideType,defaultTestUrl,+ defaultTotpType,+ heading,onStartBackgroundTest,}: Props) {const credentialGetter = useCredentialGetter();@@ -703,6 +707,7 @@ function CredentialsModal({setPasswordCredentialValues((prev) => ({...prev,name: defaultName,+ totp_type: defaultTotpType ?? prev.totp_type,}));setCreditCardCredentialValues((prev) => ({...prev,@@ -717,7 +722,14 @@ function CredentialsModal({}}// eslint-disable-next-line react-hooks/exhaustive-deps -- ref-guarded one-time init; reset and the setters are intentionally omitted so a background refetch can't re-run it- }, [isOpen, credentials, isEditMode, editingCredential, defaultTestUrl]);+ }, [+ isOpen,+ credentials,+ isEditMode,+ editingCredential,+ defaultTestUrl,+ defaultTotpType,+ ]);function reset() {setVaultType("default");@@ -2222,7 +2234,7 @@ function CredentialsModal({<DialogContent className="max-h-[90vh] w-[700px] max-w-[700px] overflow-y-auto [&::-webkit-scrollbar-thumb]:rounded-full [&::-webkit-scrollbar-thumb]:border-2 [&::-webkit-scrollbar-thumb]:border-slate-100 [&::-webkit-scrollbar-thumb]:bg-slate-300 dark:[&::-webkit-scrollbar-thumb]:border-slate-800 dark:[&::-webkit-scrollbar-thumb]:bg-slate-600 [&::-webkit-scrollbar-track]:bg-slate-100 dark:[&::-webkit-scrollbar-track]:bg-slate-800 [&::-webkit-scrollbar]:w-2"><DialogHeader><DialogTitle className="font-bold">- {isEditMode ? "Edit Credential" : "Add Credential"}+ {isEditMode ? "Edit Credential" : (heading ?? "Add Credential")}</DialogTitle></DialogHeader>{isEditMode && editingGroups.values && (diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsxindex 7e42de134..f9ee1021d 100644--- a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx@@ -16,6 +16,8 @@ const mutateMock = vi.fn();const modalState = vi.hoisted(() => ({overrideType: null as string | null,defaultTestUrl: null as string | null,+ defaultTotpType: null as string | null,+ heading: null as string | null,}));vi.mock("@/routes/browserSessions/hooks/useProcessRecordingMutation", () => ({@@ -32,14 +34,20 @@ vi.mock("@/routes/credentials/CredentialsModal", () => ({onCredentialCreated,overrideType,defaultTestUrl,+ defaultTotpType,+ heading,}: {isOpen?: boolean;onCredentialCreated?: (id: string, name?: string) => void;overrideType?: string;defaultTestUrl?: string;+ defaultTotpType?: string;+ heading?: string;}) => {modalState.overrideType = overrideType ?? null;modalState.defaultTestUrl = defaultTestUrl ?? null;+ modalState.defaultTotpType = defaultTotpType ?? null;+ modalState.heading = heading ?? null;return isOpen ? (<buttontype="button"@@ -85,6 +93,8 @@ describe("RecordingPanel", () => {mutateMock.mockReset();modalState.overrideType = null;modalState.defaultTestUrl = null;+ modalState.defaultTotpType = null;+ modalState.heading = null;useRecordingStore.setState(initialRecording, true);useWorkflowPanelStore.setState(initialPanel, true);useRecordedBlocksStore.setState(initialRecordedBlocks, true);@@ -187,9 +197,7 @@ describe("RecordingPanel", () => {expect(screen.getByText("Fill password")).toBeTruthy();expect(screen.getByText("Fill email")).toBeTruthy();- expect(- screen.getByRole("button", { name: /add to credentials/i }),- ).toBeTruthy();+ expect(screen.getByRole("button", { name: /add password/i })).toBeTruthy();expect(screen.getByRole("button", { name: /^dismiss$/i })).toBeTruthy();});@@ -206,9 +214,7 @@ describe("RecordingPanel", () => {render(<RecordingPanel browserSessionId="pbs_123" />);fireEvent.click(screen.getByRole("button", { name: /^dismiss$/i }));- expect(- screen.queryByRole("button", { name: /add to credentials/i }),- ).toBeNull();+ expect(screen.queryByRole("button", { name: /add password/i })).toBeNull();expect(screen.getByText("Fill password")).toBeTruthy();});@@ -224,19 +230,68 @@ describe("RecordingPanel", () => {});render(<RecordingPanel browserSessionId="pbs_123" />);- fireEvent.click(- screen.getByRole("button", { name: /add to credentials/i }),- );+ fireEvent.click(screen.getByRole("button", { name: /add password/i }));expect(modalState.overrideType).toBe("password");+ expect(modalState.heading).toBe("Add Password");expect(modalState.defaultTestUrl).toBe("https://example.com/login");expect(screen.getByTestId("mock-create-credential")).toBeTruthy();fireEvent.click(screen.getByTestId("mock-create-credential"));- expect(- screen.queryByRole("button", { name: /add to credentials/i }),- ).toBeNull();+ expect(screen.queryByRole("button", { name: /add password/i })).toBeNull();expect(screen.queryByTestId("mock-create-credential")).toBeNull();});++ it.each([+ {+ kind: "credit_card" as const,+ button: /add credit card/i,+ overrideType: "credit-card",+ heading: "Add Credit Card",+ defaultTotpType: null,+ },+ {+ kind: "secret" as const,+ button: /add secret/i,+ overrideType: "secret",+ heading: "Add Secret",+ defaultTotpType: null,+ },+ {+ kind: "totp" as const,+ button: /add two-factor authentication/i,+ overrideType: "password",+ heading: "Add Two-Factor Authentication",+ defaultTotpType: "authenticator",+ },+ {+ kind: "magic_link" as const,+ button: /add magic link/i,+ overrideType: "password",+ heading: "Add Magic Link",+ defaultTotpType: "email",+ },+ ])(+ "opens the $heading popup for a $kind draft",+ ({ kind, button, overrideType, heading, defaultTotpType }) => {+ useRecordingStore.setState({+ draftSteps: [+ inputDraft("secret-step", {+ credential_kind: kind,+ title: `Fill ${kind}`,+ url: "https://example.com/pay",+ }),+ ],+ });++ render(<RecordingPanel browserSessionId="pbs_123" />);+ fireEvent.click(screen.getByRole("button", { name: button }));++ expect(modalState.overrideType).toBe(overrideType);+ expect(modalState.heading).toBe(heading);+ expect(modalState.defaultTotpType).toBe(defaultTotpType);+ expect(modalState.defaultTestUrl).toBe("https://example.com/pay");+ },+ );});diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsxindex 7f4295b52..94802b0d3 100644--- a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx@@ -137,12 +137,51 @@ function StepScreenshot({ screenshot }: { screenshot: RecordingScreenshot }) {);}-function credentialModalTypeForKind(+function credentialPromptForKind(kind: NonNullable<RecordingDraftStep["credential_kind"]>,-): CredentialModalType {- return kind === "credit_card"- ? CredentialModalTypes.CREDIT_CARD- : CredentialModalTypes.PASSWORD;+): {+ type: CredentialModalType;+ defaultTotpType?: "authenticator" | "email";+ heading: string;+ buttonLabel: string;+} {+ switch (kind) {+ case "credit_card":+ return {+ type: CredentialModalTypes.CREDIT_CARD,+ heading: "Add Credit Card",+ buttonLabel: "Add credit card",+ };+ case "secret":+ return {+ type: CredentialModalTypes.SECRET,+ heading: "Add Secret",+ buttonLabel: "Add secret",+ };+ case "totp":+ return {+ type: CredentialModalTypes.PASSWORD,+ defaultTotpType: "authenticator",+ heading: "Add Two-Factor Authentication",+ buttonLabel: "Add two-factor authentication",+ };+ case "magic_link":+ return {+ type: CredentialModalTypes.PASSWORD,+ defaultTotpType: "email",+ heading: "Add Magic Link",+ buttonLabel: "Add magic link",+ };+ }++ // "password", plus any kind a backend one deploy ahead sends that this bundle has never+ // seen. This runs during render, so falling off the end would throw on `.buttonLabel`+ // and unmount the editor rather than degrade the single button.+ return {+ type: CredentialModalTypes.PASSWORD,+ heading: "Add Password",+ buttonLabel: "Add password",+ };}function DraftStepCard({@@ -153,6 +192,7 @@ function DraftStepCard({onDelete,onRename,showCredentialPrompt,+ addCredentialLabel,onAddCredentials,onDismissCredential,}: {@@ -163,6 +203,7 @@ function DraftStepCard({onDelete: () => void;onRename: (value: string) => void;showCredentialPrompt: boolean;+ addCredentialLabel: string;onAddCredentials: () => void;onDismissCredential: () => void;}) {@@ -294,7 +335,7 @@ function DraftStepCard({className="h-7"onClick={onAddCredentials}>- Add to credentials+ {addCredentialLabel}</Button><Buttontype="button"@@ -380,6 +421,8 @@ function RecordingPanel({ browserSessionId }: Props) {testUrl: string | null;url: string | null;stepId: string;+ defaultTotpType?: "authenticator" | "email";+ heading: string;} | null>(null);const feedEndRef = useRef<HTMLDivElement | null>(null);const committedRef = useRef(false);@@ -605,12 +648,20 @@ function RecordingPanel({ browserSessionId }: Props) {Boolean(step.credential_kind) &&!dismissedCredentialStepIds.includes(step.step_id)}+ addCredentialLabel={+ step.credential_kind+ ? credentialPromptForKind(step.credential_kind).buttonLabel+ : "Add to credentials"+ }onAddCredentials={() => {if (!step.credential_kind) {return;}+ const prompt = credentialPromptForKind(step.credential_kind);setCredentialModal({- type: credentialModalTypeForKind(step.credential_kind),+ type: prompt.type,+ defaultTotpType: prompt.defaultTotpType,+ heading: prompt.heading,testUrl: step.url ?? null,url: step.url ?? null,stepId: step.step_id,@@ -744,6 +795,8 @@ function RecordingPanel({ browserSessionId }: Props) {isOpenoverrideType={credentialModal.type}defaultTestUrl={credentialModal.testUrl ?? undefined}+ defaultTotpType={credentialModal.defaultTotpType}+ heading={credentialModal.heading}onOpenChange={(open) => {if (!open) {setCredentialModal(null);diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.tsindex f0ca1e33d..721ab4138 100644--- a/skyvern-frontend/src/store/useRecordingStore.ts+++ b/skyvern-frontend/src/store/useRecordingStore.ts@@ -127,7 +127,13 @@ export interface RecordingDraftStep {parameter_keys: Array<string>;timestamp_start?: number | null;timestamp_end?: number | null;- credential_kind?: "password" | "totp" | "credit_card" | null;+ credential_kind?:+ | "password"+ | "totp"+ | "credit_card"+ | "secret"+ | "magic_link"+ | null;}export interface RecordingInterpretationUpdate {diff --git a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.jsindex 692e3a81d..aa4ded849 100644--- a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js+++ b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js@@ -125,12 +125,56 @@"current-password","new-password","one-time-code",+ "cc-name","cc-number","cc-csc","cc-exp","cc-exp-month","cc-exp-year",]);+ const CREDIT_CARD_HINT_PHRASES = [+ "card number",+ "credit card",+ "cardholder",+ "cvv",+ "cvc",+ ];+ const TOTP_HINT_PHRASES = [+ "otp",+ "totp",+ "2fa",+ "two factor",+ "one time code",+ "verification code",+ "authenticator code",+ ];+ const SECRET_HINT_PHRASES = [+ "api key",+ "apikey",+ "access token",+ "client secret",+ "webhook secret",+ "private key",+ "bearer token",+ "secret value",+ ];++ const normalizeHaystack = (...parts) => {+ const joined = parts.filter(Boolean).join(" ").toLowerCase();+ const normalized = joined.replace(/[^a-z0-9]+/g, " ").trim();+ return normalized ? ` ${normalized} ` : "";+ };++ const haystackHasPhrase = (haystack, phrase) => {+ if (!haystack) {+ return false;+ }+ const needle = ` ${String(phrase)+ .toLowerCase()+ .replace(/[^a-z0-9]+/g, " ")+ .trim()} `;+ return haystack.includes(needle);+ };const isSecretField = (element) => {if (!element) {@@ -143,9 +187,32 @@const autocomplete = String((element.getAttribute && element.getAttribute("autocomplete")) || "",).toLowerCase();- return autocomplete- .split(/\s+/)- .some((token) => SECRET_AUTOCOMPLETE_TOKENS.has(token));+ if (+ autocomplete+ .split(/\s+/)+ .some((token) => SECRET_AUTOCOMPLETE_TOKENS.has(token))+ ) {+ return true;+ }+ const haystack = normalizeHaystack(+ element.id,+ element.name,+ element.getAttribute && element.getAttribute("name"),+ element.getAttribute && element.getAttribute("aria-label"),+ element.getAttribute && element.getAttribute("placeholder"),+ element.getAttribute && element.getAttribute("title"),+ );+ return (+ CREDIT_CARD_HINT_PHRASES.some((phrase) =>+ haystackHasPhrase(haystack, phrase),+ ) ||+ TOTP_HINT_PHRASES.some((phrase) =>+ haystackHasPhrase(haystack, phrase),+ ) ||+ SECRET_HINT_PHRASES.some((phrase) =>+ haystackHasPhrase(haystack, phrase),+ )+ );};const IMPLICIT_INPUT_ROLES = {diff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.pyindex 468bf6ae1..f261f1caf 100644--- a/skyvern/services/browser_recording/interpretation.py+++ b/skyvern/services/browser_recording/interpretation.py@@ -25,8 +25,10 @@ from skyvern.services.browser_recording.service import (from skyvern.services.browser_recording.types import (Action,ActionBlockable,+ ActionClick,ActionInputText,ActionKind,+ CredentialKind,ExfiltratedCdpEvent,ExfiltratedConsoleEvent,ExfiltratedEvent,@@ -111,10 +113,17 @@ def _extra_field(block: OutputBlock, field_name: str, fallback: t.Any) -> t.Any:return fallback-def _credential_kind_for_action(action: Action) -> t.Literal["password", "totp", "credit_card"] | None:- if not isinstance(action, ActionInputText):+def _credential_kind_for_action(action: Action) -> CredentialKind | None:+ if not isinstance(action, (ActionClick, ActionInputText)):return None- return credential_kind_for_target(action.target.input_type, action.target.autocomplete)+ return credential_kind_for_target(+ action.target.input_type,+ action.target.autocomplete,+ field_id=action.target.id,+ accessible_name=action.target.accessible_name,+ texts=action.target.texts,+ tag_name=action.target.tag_name,+ )def _draft_step_from_block(diff --git a/skyvern/services/browser_recording/redact.py b/skyvern/services/browser_recording/redact.pyindex 33cb07956..2cda34e69 100644--- a/skyvern/services/browser_recording/redact.py+++ b/skyvern/services/browser_recording/redact.py@@ -6,11 +6,10 @@ Keep the token lists identical to `isSecretField` infrom __future__ import annotations+import reimport typing as t-from skyvern.services.browser_recording.types import ExfiltratedConsoleEvent, ExfiltratedEvent--CredentialKind = t.Literal["password", "totp", "credit_card"]+from skyvern.services.browser_recording.types import CredentialKind, ExfiltratedConsoleEvent, ExfiltratedEventSECRET_INPUT_TYPES = frozenset({"password"})SECRET_AUTOCOMPLETE_TOKENS = frozenset(@@ -18,6 +17,7 @@ SECRET_AUTOCOMPLETE_TOKENS = frozenset("current-password","new-password","one-time-code",+ "cc-name","cc-number","cc-csc","cc-exp",@@ -28,6 +28,7 @@ SECRET_AUTOCOMPLETE_TOKENS = frozenset(PASSWORD_AUTOCOMPLETE_TOKENS = frozenset({"current-password", "new-password"})CREDIT_CARD_AUTOCOMPLETE_TOKENS = frozenset({+ "cc-name","cc-number","cc-csc","cc-exp",@@ -36,6 +37,45 @@ CREDIT_CARD_AUTOCOMPLETE_TOKENS = frozenset(})+# Phrase matchers run on a punctuation-normalized haystack of id / name / labels.+CREDIT_CARD_HINT_PHRASES = (+ "card number",+ "credit card",+ "cardholder",+ "cvv",+ "cvc",+)+TOTP_HINT_PHRASES = (+ "otp",+ "totp",+ "2fa",+ "two factor",+ "one time code",+ "verification code",+ "authenticator code",+)+SECRET_HINT_PHRASES = (+ "api key",+ "apikey",+ "access token",+ "client secret",+ "webhook secret",+ "private key",+ "bearer token",+ "secret value",+)+MAGIC_LINK_HINT_PHRASES = (+ "magic link",+ "email me a link",+ "send login link",+ "send sign in link",+ "passwordless",+)++_NON_ALNUM_RE = re.compile(r"[^a-z0-9]+")++HaystackPart = str | None | list[str]+def _autocomplete_tokens(autocomplete: str | None) -> list[str]:if not autocomplete:@@ -43,20 +83,87 @@ def _autocomplete_tokens(autocomplete: str | None) -> list[str]:return autocomplete.lower().split()-def is_secret_field(input_type: str | None, autocomplete: str | None) -> bool:- if (input_type or "").lower() in SECRET_INPUT_TYPES:- return True- return any(token in SECRET_AUTOCOMPLETE_TOKENS for token in _autocomplete_tokens(autocomplete))---def credential_kind_for_target(input_type: str | None, autocomplete: str | None) -> CredentialKind | None:+# `texts` comes from getElementText, which appends innerText/textContent for any element+# that has children: for a <select> that is the option labels, for a container it is+# arbitrary page copy. <input> is void, so only there is `texts` purely labelling. Anything+# else is classified from its id and accessible name only, which is also what the page+# script's own matcher reads.+TEXT_IS_LABEL_TAGS = frozenset({"input"})+++def texts_are_labels(tag_name: str | None) -> bool:+ return (tag_name or "").lower() in TEXT_IS_LABEL_TAGS+++def _normalized_haystack(*parts: HaystackPart) -> str:+ chunks: list[str] = []+ for part in parts:+ if part is None:+ continue+ if isinstance(part, list):+ chunks.extend(item for item in part if item)+ elif part:+ chunks.append(part)+ if not chunks:+ return ""+ return " " + _NON_ALNUM_RE.sub(" ", " ".join(chunks).lower()).strip() + " "+++def _haystack_has_phrase(haystack: str, phrase: str) -> bool:+ if not haystack:+ return False+ needle = " " + _NON_ALNUM_RE.sub(" ", phrase.lower()).strip() + " "+ return needle in haystack+++def _haystack_has_any(haystack: str, phrases: t.Sequence[str]) -> bool:+ return any(_haystack_has_phrase(haystack, phrase) for phrase in phrases)+++def is_secret_field(+ input_type: str | None,+ autocomplete: str | None,+ *,+ field_id: str | None = None,+ accessible_name: str | None = None,+ texts: list[str] | None = None,+ tag_name: str | None = None,+) -> bool:+ kind = credential_kind_for_target(+ input_type,+ autocomplete,+ field_id=field_id,+ accessible_name=accessible_name,+ texts=texts,+ tag_name=tag_name,+ )+ return kind in {"password", "totp", "credit_card", "secret"}+++def credential_kind_for_target(+ input_type: str | None,+ autocomplete: str | None,+ *,+ field_id: str | None = None,+ accessible_name: str | None = None,+ texts: list[str] | None = None,+ tag_name: str | None = None,+) -> CredentialKind | None:tokens = _autocomplete_tokens(autocomplete)- if any(token in CREDIT_CARD_AUTOCOMPLETE_TOKENS for token in tokens):+ haystack = _normalized_haystack(field_id, accessible_name, texts if texts_are_labels(tag_name) else None)++ if any(token in CREDIT_CARD_AUTOCOMPLETE_TOKENS for token in tokens) or _haystack_has_any(+ haystack, CREDIT_CARD_HINT_PHRASES+ ):return "credit_card"- if "one-time-code" in tokens:+ if "one-time-code" in tokens or _haystack_has_any(haystack, TOTP_HINT_PHRASES):return "totp"+ if _haystack_has_any(haystack, SECRET_HINT_PHRASES):+ return "secret"if (input_type or "").lower() == "password" or any(token in PASSWORD_AUTOCOMPLETE_TOKENS for token in tokens):return "password"+ if _haystack_has_any(haystack, MAGIC_LINK_HINT_PHRASES):+ return "magic_link"return None@@ -74,7 +181,14 @@ def redact_console_event(event: ExfiltratedEvent) -> ExfiltratedEvent:return eventtarget = event.params.target- if not is_secret_field(target.inputType, target.autocomplete):+ if not is_secret_field(+ target.inputType,+ target.autocomplete,+ field_id=target.id,+ accessible_name=target.accessibleName,+ texts=target.text,+ tag_name=target.tagName,+ ):return eventtarget.value = Nonediff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.pyindex fe3416327..f3ab55cd1 100644--- a/skyvern/services/browser_recording/service.py+++ b/skyvern/services/browser_recording/service.py@@ -591,7 +591,14 @@ class Processor:prompt_name = "recording-action-block-prompt-input-text"if (isinstance(action, ActionInputText)- and is_secret_field(action.target.input_type, action.target.autocomplete)+ and is_secret_field(+ action.target.input_type,+ action.target.autocomplete,+ field_id=action.target.id,+ accessible_name=action.target.accessible_name,+ texts=action.target.texts,+ tag_name=action.target.tag_name,+ )and action.input_value):action = action.model_copy(update={"input_value": ""})diff --git a/skyvern/services/browser_recording/state_machines/input_text.py b/skyvern/services/browser_recording/state_machines/input_text.pyindex b0b1a2780..41ce624c7 100644--- a/skyvern/services/browser_recording/state_machines/input_text.py+++ b/skyvern/services/browser_recording/state_machines/input_text.py@@ -104,7 +104,14 @@ class StateMachineInputText(StateMachine):LOG.debug("~ emitting input text action")- secret = is_secret_field(self.target.inputType, self.target.autocomplete)+ secret = is_secret_field(+ self.target.inputType,+ self.target.autocomplete,+ field_id=self.target.id,+ accessible_name=self.target.accessibleName,+ texts=self.target.text,+ tag_name=self.target.tagName,+ )input_value = event.params.target.valueif input_value is None and not secret:diff --git a/skyvern/services/browser_recording/types.py b/skyvern/services/browser_recording/types.pyindex 71de88ccd..ef86dba3f 100644--- a/skyvern/services/browser_recording/types.py+++ b/skyvern/services/browser_recording/types.py@@ -78,6 +78,8 @@ Action = ActionClick | ActionHover | ActionInputText | ActionUrlChange | ActionWActionBlockable = ActionClick | ActionHover | ActionInputText+CredentialKind = Literal["password", "totp", "credit_card", "secret", "magic_link"]+class ActionTarget(BaseModel):class_name: str | None = None@@ -134,7 +136,7 @@ class RecordingDraftStep(BaseModel):# locally-captured artifacts (e.g. stream screenshots) with each step.timestamp_start: float | None = Nonetimestamp_end: float | None = None- credential_kind: Literal["password", "totp", "credit_card"] | None = None+ credential_kind: CredentialKind | None = Noneclass RecordingInterpretationUpdate(BaseModel):diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.pyindex 7506f4db2..2694ef969 100644--- a/tests/unit/services/test_browser_recording.py+++ b/tests/unit/services/test_browser_recording.py@@ -5,12 +5,15 @@ Just an example unit test for now. Will expand later.import asyncioimport base64import gzip+import reimport timeimport typing as timport zlib+from pathlib import Pathimport pytest+import skyvernfrom skyvern.forge import appfrom skyvern.forge.sdk.routes.streaming.channels.exfiltration import ExfiltratedEvent as StreamingExfiltratedEventfrom skyvern.forge.sdk.routes.streaming.channels.exfiltration import (@@ -19,6 +22,7 @@ from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (ExfiltrationChannel,)+from skyvern.services.browser_recording import redact as redact_modulefrom skyvern.services.browser_recording.interpretation import RecordingInterpretationSessionfrom skyvern.services.browser_recording.service import (DUPLICATE_ACTION_WINDOW_MS,@@ -236,12 +240,26 @@ def _empty_action_target() -> ActionTarget:return ActionTarget(mouse=Mouse(xp=None, yp=None))-def make_streaming_console_click(timestamp_ms: float) -> StreamingExfiltratedEvent:+def make_streaming_console_click(+ timestamp_ms: float,+ *,+ target_id: str = "button-1",+ target_text: str = "Click me",+ accessible_name: str | None = None,+) -> StreamingExfiltratedEvent:+ target: dict[str, t.Any] = {+ "id": target_id,+ "skyId": "sky-123",+ "tagName": "BUTTON",+ "text": [target_text],+ }+ if accessible_name is not None:+ target["accessibleName"] = accessible_namereturn StreamingExfiltratedEvent(event_name="user_interaction",params={"type": "click",- "target": {"id": "button-1", "skyId": "sky-123", "tagName": "BUTTON", "text": ["Click me"]},+ "target": target,"timestamp": timestamp_ms,"url": "https://example.com","activeElement": {"tagName": "BUTTON"},@@ -270,6 +288,7 @@ def make_streaming_console_input(target_text: str = "Email",input_type: str | None = None,autocomplete: str | None = None,+ accessible_name: str | None = None,) -> list[StreamingExfiltratedEvent]:target: dict[str, t.Any] = {"id": target_id,@@ -282,6 +301,8 @@ def make_streaming_console_input(target["inputType"] = input_typeif autocomplete is not None:target["autocomplete"] = autocomplete+ if accessible_name is not None:+ target["accessibleName"] = accessible_namecommon = {"target": target,"timestamp": timestamp_ms,@@ -921,9 +942,28 @@ def test_is_secret_field_and_credential_kind() -> None:assert credential_kind_for_target("text", "current-password") == "password"assert credential_kind_for_target("text", "one-time-code") == "totp"assert credential_kind_for_target("text", "cc-number") == "credit_card"+ assert credential_kind_for_target("text", "cc-name") == "credit_card"assert credential_kind_for_target("email", None) is Noneassert credential_kind_for_target("text", "off") is None+ assert credential_kind_for_target("text", None, field_id="otp", texts=["Verification code"]) == "totp"+ assert credential_kind_for_target("password", None, accessible_name="API Key") == "secret"+ assert credential_kind_for_target("text", None, texts=["Card number"], tag_name="input") == "credit_card"++ # `texts` carries innerText for elements with children, so it is only a label on <input>.+ # A <select> whose options happen to contain a hint phrase must keep its recorded value.+ assert credential_kind_for_target("text", None, texts=["Card number"], tag_name="select") is None+ assert (+ credential_kind_for_target("text", None, texts=["Reason: Verification code issue"], tag_name="select") is None+ )+ assert credential_kind_for_target(None, None, texts=["Two factor authentication"], tag_name="div") is None+ # The label itself still classifies, whatever the tag.+ assert credential_kind_for_target("text", None, accessible_name="Verification code", tag_name="select") == "totp"+ assert credential_kind_for_target("password", None, accessible_name="Password") == "password"+ assert credential_kind_for_target(None, None, accessible_name="Email me a magic link") == "magic_link"+ assert is_secret_field("password", None, accessible_name="API Key") is True+ assert is_secret_field(None, None, accessible_name="Email me a magic link") is False+def test_reify_strips_secret_values_from_console_events() -> None:processor = Processor(PBS_ID, ORG_ID, WP_ID)@@ -992,6 +1032,41 @@ def test_password_fill_still_emits_empty_input_text() -> None:assert action.target.input_type == "password"+def _js_string_literals(source: str, name: str) -> set[str]:+ match = re.search(rf"const {name} = (?:new Set\()?\[(.*?)\]", source, re.DOTALL)+ assert match, f"{name} not found in exfiltrate.js"+ return set(re.findall(r'"([^"]*)"', match.group(1)))+++def test_secret_matcher_lists_match_between_js_and_python() -> None:+ """The page script and the Python ingest must classify the same fields as secret.++ A token added to one side only silently changes what gets captured, so parity is+ asserted here rather than left to the docstring in redact.py.+ """+ js_source = (Path(skyvern.__file__).parent / "forge/sdk/routes/streaming/channels/js/exfiltrate.js").read_text()+ matcher = re.search(r"const isSecretField = \(element\) => \{(.*?)\n \};", js_source, re.DOTALL)+ assert matcher, "isSecretField not found in exfiltrate.js"+ js_matcher_body = matcher.group(1)++ # MAGIC_LINK_HINT_PHRASES is deliberately absent from the JS: "magic_link" is not a+ # secret kind, so it labels a field without redacting it.+ for name, python_value in (+ ("SECRET_INPUT_TYPES", redact_module.SECRET_INPUT_TYPES),+ ("SECRET_AUTOCOMPLETE_TOKENS", redact_module.SECRET_AUTOCOMPLETE_TOKENS),+ ("CREDIT_CARD_HINT_PHRASES", redact_module.CREDIT_CARD_HINT_PHRASES),+ ("TOTP_HINT_PHRASES", redact_module.TOTP_HINT_PHRASES),+ ("SECRET_HINT_PHRASES", redact_module.SECRET_HINT_PHRASES),+ ):+ assert _js_string_literals(js_source, name) == set(python_value), (+ f"{name} differs between exfiltrate.js and redact.py"+ )+ # Matching lists are worthless if the page script stops consulting one: deleting a+ # clause from isSecretField while leaving its array declared would keep the parity+ # assertion above green and silently stop redacting that family at source.+ assert name in js_matcher_body, f"{name} is declared but never used by isSecretField"++def test_password_submitted_with_enter_still_emits_input_text() -> None:"""Enter-to-submit never blurs before navigating, so the Enter keydown is the only emit signal.@@ -1039,6 +1114,37 @@ def test_secret_character_keystrokes_still_redacted() -> None:assert submit.params.target.value is None+def test_phrase_only_secret_has_its_value_stripped_through_the_processor() -> None:+ """A field classified *only* by its label must lose its value, not just get a kind.++ The password/OTP cases are carried by input_type/autocomplete, so neither proves the+ phrase matching this PR adds actually reaches redaction.+ """+ target = {+ "id": "field-1",+ "tagName": "INPUT",+ "inputType": "text",+ "autocomplete": None,+ "accessibleName": "API Key",+ "value": "sk_live_not_real",+ }+ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ events = processor.reify(+ [+ make_focus_event(target=target, timestamp=1000.0).model_dump(),+ make_keydown_event(target=target, timestamp=1001.0, key="s").model_dump(),+ make_blur_event(target=target, timestamp=1002.0).model_dump(),+ ]+ )+ actions = processor.events_to_actions(events)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, ActionInputText)+ assert action.input_value == ""+ assert "sk_live_not_real" not in repr(action)++def test_non_secret_empty_value_still_skipped() -> None:target = {"id": "email",@@ -1186,3 +1292,69 @@ async def test_otp_autocomplete_ingest_redacts_and_stamps_totp(assert len(steps) == 1assert steps[0].credential_kind == "totp"assert "654321" not in (steps[0].navigation_goal or "")+++async def test_api_key_ingest_stamps_secret_kind(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:+ raise RuntimeError("llm unavailable")++ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)++ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda update: None,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ )+ session.ingest_events(+ make_streaming_console_input(+ timestamp_ms=1000.0,+ input_value="sk_live_secret",+ target_id="api-key",+ target_text="API Key",+ input_type="password",+ accessible_name="API Key",+ )+ )+ steps = await session.flush()++ assert len(steps) == 1+ assert steps[0].credential_kind == "secret"+ assert "sk_live_secret" not in (steps[0].navigation_goal or "")+++async def test_magic_link_click_stamps_kind(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:+ raise RuntimeError("llm unavailable")++ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)++ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda update: None,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ )+ session.ingest_events(+ [+ make_streaming_console_click(+ 1000.0,+ target_text="Email me a magic link",+ accessible_name="Email me a magic link",+ )+ ]+ )+ steps = await session.flush()++ assert len(steps) == 1+ assert steps[0].credential_kind == "magic_link"