Skip to content

Automate browser based workflows with AI

Show the matching credentials popup from Record Browser (#SKY-15140) (#8323)

Cindy Li1287a6d10d2c
diff --git a/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx b/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx
index 2550dad37..664ed6015 100644
--- a/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx
+++ b/skyvern-frontend/src/routes/credentials/CredentialsModal.test.tsx
@@ -1378,3 +1378,77 @@ describe("CredentialsModal run-sequentially toggle", () => {
);
}, 10_000);
});
+
+describe("CredentialsModal record-browser defaults", () => {
+ function renderRecordingModal(opts: {
+ overrideType?: (typeof CredentialModalTypes)[keyof typeof CredentialModalTypes];
+ defaultTotpType?: "authenticator" | "email";
+ heading?: string;
+ }) {
+ const queryClient = new QueryClient({
+ defaultOptions: {
+ queries: { retry: false },
+ mutations: { retry: false },
+ },
+ });
+ return render(
+ <QueryClientProvider client={queryClient}>
+ <MemoryRouter>
+ <CredentialsModal
+ isOpen
+ onOpenChange={vi.fn()}
+ overrideType={opts.overrideType ?? CredentialModalTypes.PASSWORD}
+ defaultTotpType={opts.defaultTotpType}
+ heading={opts.heading}
+ />
+ </MemoryRouter>
+ </QueryClientProvider>,
+ );
+ }
+
+ it("uses the supplied heading and opens 2FA on the authenticator method", async () => {
+ renderRecordingModal({
+ defaultTotpType: "authenticator",
+ heading: "Add Two-Factor Authentication",
+ });
+
+ expect(
+ await screen.findByRole("heading", {
+ name: "Add Two-Factor Authentication",
+ }),
+ ).toBeTruthy();
+ // Every method tile renders whenever the accordion is open, so presence alone passes
+ // for defaultTotpType "email" too; aria-pressed is what distinguishes the selection.
+ const authenticatorMethod = await screen.findByRole("button", {
+ name: /authenticator app/i,
+ });
+ expect(authenticatorMethod.getAttribute("aria-pressed")).toBe("true");
+ const emailMethod = await screen.findByRole("button", { name: /email/i });
+ expect(emailMethod.getAttribute("aria-pressed")).toBe("false");
+ });
+
+ it("opens 2FA on the email method for a magic-link prompt", async () => {
+ renderRecordingModal({
+ defaultTotpType: "email",
+ heading: "Add Magic Link",
+ });
+
+ expect(
+ await screen.findByRole("heading", { name: "Add Magic Link" }),
+ ).toBeTruthy();
+ const emailMethod = await screen.findByRole("button", { name: /email/i });
+ expect(emailMethod.getAttribute("aria-pressed")).toBe("true");
+ });
+
+ it("renders the secret form for a secret override", async () => {
+ renderRecordingModal({
+ overrideType: CredentialModalTypes.SECRET,
+ heading: "Add Secret",
+ });
+
+ expect(
+ await screen.findByRole("heading", { name: "Add Secret" }),
+ ).toBeTruthy();
+ expect(screen.getByText("Secret Label (optional)")).toBeTruthy();
+ });
+});
diff --git a/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx b/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx
index 9504e9a79..c7b342cc9 100644
--- a/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx
+++ b/skyvern-frontend/src/routes/credentials/CredentialsModal.tsx
@@ -258,6 +258,8 @@ type Props = {
// tested_url and matches later hostname-keyed asks. Create mode only; leaving
// it undefined (every non-copilot caller) keeps the field empty as before.
defaultTestUrl?: string;
+ defaultTotpType?: PasswordCredential["totp_type"];
+ heading?: string;
/** Called after a credential is saved with "Save browser session" checked to trigger an async test */
onStartBackgroundTest?: (
credentialId: string,
@@ -295,6 +297,8 @@ function CredentialsModal({
editingCredential,
overrideType,
defaultTestUrl,
+ defaultTotpType,
+ heading,
onStartBackgroundTest,
}: Props) {
const credentialGetter = useCredentialGetter();
@@ -703,6 +707,7 @@ function CredentialsModal({
setPasswordCredentialValues((prev) => ({
...prev,
name: defaultName,
+ totp_type: defaultTotpType ?? prev.totp_type,
}));
setCreditCardCredentialValues((prev) => ({
...prev,
@@ -717,7 +722,14 @@ function CredentialsModal({
}
}
// eslint-disable-next-line react-hooks/exhaustive-deps -- ref-guarded one-time init; reset and the setters are intentionally omitted so a background refetch can't re-run it
- }, [isOpen, credentials, isEditMode, editingCredential, defaultTestUrl]);
+ }, [
+ isOpen,
+ credentials,
+ isEditMode,
+ editingCredential,
+ defaultTestUrl,
+ defaultTotpType,
+ ]);
function reset() {
setVaultType("default");
@@ -2222,7 +2234,7 @@ function CredentialsModal({
<DialogContent className="max-h-[90vh] w-[700px] max-w-[700px] overflow-y-auto [&::-webkit-scrollbar-thumb]:rounded-full [&::-webkit-scrollbar-thumb]:border-2 [&::-webkit-scrollbar-thumb]:border-slate-100 [&::-webkit-scrollbar-thumb]:bg-slate-300 dark:[&::-webkit-scrollbar-thumb]:border-slate-800 dark:[&::-webkit-scrollbar-thumb]:bg-slate-600 [&::-webkit-scrollbar-track]:bg-slate-100 dark:[&::-webkit-scrollbar-track]:bg-slate-800 [&::-webkit-scrollbar]:w-2">
<DialogHeader>
<DialogTitle className="font-bold">
- {isEditMode ? "Edit Credential" : "Add Credential"}
+ {isEditMode ? "Edit Credential" : (heading ?? "Add Credential")}
</DialogTitle>
</DialogHeader>
{isEditMode && editingGroups.values && (
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx
index 7e42de134..f9ee1021d 100644
--- a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx
@@ -16,6 +16,8 @@ const mutateMock = vi.fn();
const modalState = vi.hoisted(() => ({
overrideType: null as string | null,
defaultTestUrl: null as string | null,
+ defaultTotpType: null as string | null,
+ heading: null as string | null,
}));
vi.mock("@/routes/browserSessions/hooks/useProcessRecordingMutation", () => ({
@@ -32,14 +34,20 @@ vi.mock("@/routes/credentials/CredentialsModal", () => ({
onCredentialCreated,
overrideType,
defaultTestUrl,
+ defaultTotpType,
+ heading,
}: {
isOpen?: boolean;
onCredentialCreated?: (id: string, name?: string) => void;
overrideType?: string;
defaultTestUrl?: string;
+ defaultTotpType?: string;
+ heading?: string;
}) => {
modalState.overrideType = overrideType ?? null;
modalState.defaultTestUrl = defaultTestUrl ?? null;
+ modalState.defaultTotpType = defaultTotpType ?? null;
+ modalState.heading = heading ?? null;
return isOpen ? (
<button
type="button"
@@ -85,6 +93,8 @@ describe("RecordingPanel", () => {
mutateMock.mockReset();
modalState.overrideType = null;
modalState.defaultTestUrl = null;
+ modalState.defaultTotpType = null;
+ modalState.heading = null;
useRecordingStore.setState(initialRecording, true);
useWorkflowPanelStore.setState(initialPanel, true);
useRecordedBlocksStore.setState(initialRecordedBlocks, true);
@@ -187,9 +197,7 @@ describe("RecordingPanel", () => {
expect(screen.getByText("Fill password")).toBeTruthy();
expect(screen.getByText("Fill email")).toBeTruthy();
- expect(
- screen.getByRole("button", { name: /add to credentials/i }),
- ).toBeTruthy();
+ expect(screen.getByRole("button", { name: /add password/i })).toBeTruthy();
expect(screen.getByRole("button", { name: /^dismiss$/i })).toBeTruthy();
});
@@ -206,9 +214,7 @@ describe("RecordingPanel", () => {
render(<RecordingPanel browserSessionId="pbs_123" />);
fireEvent.click(screen.getByRole("button", { name: /^dismiss$/i }));
- expect(
- screen.queryByRole("button", { name: /add to credentials/i }),
- ).toBeNull();
+ expect(screen.queryByRole("button", { name: /add password/i })).toBeNull();
expect(screen.getByText("Fill password")).toBeTruthy();
});
@@ -224,19 +230,68 @@ describe("RecordingPanel", () => {
});
render(<RecordingPanel browserSessionId="pbs_123" />);
- fireEvent.click(
- screen.getByRole("button", { name: /add to credentials/i }),
- );
+ fireEvent.click(screen.getByRole("button", { name: /add password/i }));
expect(modalState.overrideType).toBe("password");
+ expect(modalState.heading).toBe("Add Password");
expect(modalState.defaultTestUrl).toBe("https://example.com/login");
expect(screen.getByTestId("mock-create-credential")).toBeTruthy();
fireEvent.click(screen.getByTestId("mock-create-credential"));
- expect(
- screen.queryByRole("button", { name: /add to credentials/i }),
- ).toBeNull();
+ expect(screen.queryByRole("button", { name: /add password/i })).toBeNull();
expect(screen.queryByTestId("mock-create-credential")).toBeNull();
});
+
+ it.each([
+ {
+ kind: "credit_card" as const,
+ button: /add credit card/i,
+ overrideType: "credit-card",
+ heading: "Add Credit Card",
+ defaultTotpType: null,
+ },
+ {
+ kind: "secret" as const,
+ button: /add secret/i,
+ overrideType: "secret",
+ heading: "Add Secret",
+ defaultTotpType: null,
+ },
+ {
+ kind: "totp" as const,
+ button: /add two-factor authentication/i,
+ overrideType: "password",
+ heading: "Add Two-Factor Authentication",
+ defaultTotpType: "authenticator",
+ },
+ {
+ kind: "magic_link" as const,
+ button: /add magic link/i,
+ overrideType: "password",
+ heading: "Add Magic Link",
+ defaultTotpType: "email",
+ },
+ ])(
+ "opens the $heading popup for a $kind draft",
+ ({ kind, button, overrideType, heading, defaultTotpType }) => {
+ useRecordingStore.setState({
+ draftSteps: [
+ inputDraft("secret-step", {
+ credential_kind: kind,
+ title: `Fill ${kind}`,
+ url: "https://example.com/pay",
+ }),
+ ],
+ });
+
+ render(<RecordingPanel browserSessionId="pbs_123" />);
+ fireEvent.click(screen.getByRole("button", { name: button }));
+
+ expect(modalState.overrideType).toBe(overrideType);
+ expect(modalState.heading).toBe(heading);
+ expect(modalState.defaultTotpType).toBe(defaultTotpType);
+ expect(modalState.defaultTestUrl).toBe("https://example.com/pay");
+ },
+ );
});
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx
index 7f4295b52..94802b0d3 100644
--- a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx
@@ -137,12 +137,51 @@ function StepScreenshot({ screenshot }: { screenshot: RecordingScreenshot }) {
);
}
-function credentialModalTypeForKind(
+function credentialPromptForKind(
kind: NonNullable<RecordingDraftStep["credential_kind"]>,
-): CredentialModalType {
- return kind === "credit_card"
- ? CredentialModalTypes.CREDIT_CARD
- : CredentialModalTypes.PASSWORD;
+): {
+ type: CredentialModalType;
+ defaultTotpType?: "authenticator" | "email";
+ heading: string;
+ buttonLabel: string;
+} {
+ switch (kind) {
+ case "credit_card":
+ return {
+ type: CredentialModalTypes.CREDIT_CARD,
+ heading: "Add Credit Card",
+ buttonLabel: "Add credit card",
+ };
+ case "secret":
+ return {
+ type: CredentialModalTypes.SECRET,
+ heading: "Add Secret",
+ buttonLabel: "Add secret",
+ };
+ case "totp":
+ return {
+ type: CredentialModalTypes.PASSWORD,
+ defaultTotpType: "authenticator",
+ heading: "Add Two-Factor Authentication",
+ buttonLabel: "Add two-factor authentication",
+ };
+ case "magic_link":
+ return {
+ type: CredentialModalTypes.PASSWORD,
+ defaultTotpType: "email",
+ heading: "Add Magic Link",
+ buttonLabel: "Add magic link",
+ };
+ }
+
+ // "password", plus any kind a backend one deploy ahead sends that this bundle has never
+ // seen. This runs during render, so falling off the end would throw on `.buttonLabel`
+ // and unmount the editor rather than degrade the single button.
+ return {
+ type: CredentialModalTypes.PASSWORD,
+ heading: "Add Password",
+ buttonLabel: "Add password",
+ };
}
function DraftStepCard({
@@ -153,6 +192,7 @@ function DraftStepCard({
onDelete,
onRename,
showCredentialPrompt,
+ addCredentialLabel,
onAddCredentials,
onDismissCredential,
}: {
@@ -163,6 +203,7 @@ function DraftStepCard({
onDelete: () => void;
onRename: (value: string) => void;
showCredentialPrompt: boolean;
+ addCredentialLabel: string;
onAddCredentials: () => void;
onDismissCredential: () => void;
}) {
@@ -294,7 +335,7 @@ function DraftStepCard({
className="h-7"
onClick={onAddCredentials}
>
- Add to credentials
+ {addCredentialLabel}
</Button>
<Button
type="button"
@@ -380,6 +421,8 @@ function RecordingPanel({ browserSessionId }: Props) {
testUrl: string | null;
url: string | null;
stepId: string;
+ defaultTotpType?: "authenticator" | "email";
+ heading: string;
} | null>(null);
const feedEndRef = useRef<HTMLDivElement | null>(null);
const committedRef = useRef(false);
@@ -605,12 +648,20 @@ function RecordingPanel({ browserSessionId }: Props) {
Boolean(step.credential_kind) &&
!dismissedCredentialStepIds.includes(step.step_id)
}
+ addCredentialLabel={
+ step.credential_kind
+ ? credentialPromptForKind(step.credential_kind).buttonLabel
+ : "Add to credentials"
+ }
onAddCredentials={() => {
if (!step.credential_kind) {
return;
}
+ const prompt = credentialPromptForKind(step.credential_kind);
setCredentialModal({
- type: credentialModalTypeForKind(step.credential_kind),
+ type: prompt.type,
+ defaultTotpType: prompt.defaultTotpType,
+ heading: prompt.heading,
testUrl: step.url ?? null,
url: step.url ?? null,
stepId: step.step_id,
@@ -744,6 +795,8 @@ function RecordingPanel({ browserSessionId }: Props) {
isOpen
overrideType={credentialModal.type}
defaultTestUrl={credentialModal.testUrl ?? undefined}
+ defaultTotpType={credentialModal.defaultTotpType}
+ heading={credentialModal.heading}
onOpenChange={(open) => {
if (!open) {
setCredentialModal(null);
diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.ts
index f0ca1e33d..721ab4138 100644
--- a/skyvern-frontend/src/store/useRecordingStore.ts
+++ b/skyvern-frontend/src/store/useRecordingStore.ts
@@ -127,7 +127,13 @@ export interface RecordingDraftStep {
parameter_keys: Array<string>;
timestamp_start?: number | null;
timestamp_end?: number | null;
- credential_kind?: "password" | "totp" | "credit_card" | null;
+ credential_kind?:
+ | "password"
+ | "totp"
+ | "credit_card"
+ | "secret"
+ | "magic_link"
+ | null;
}
export interface RecordingInterpretationUpdate {
diff --git a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
index 692e3a81d..aa4ded849 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
+++ b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
@@ -125,12 +125,56 @@
"current-password",
"new-password",
"one-time-code",
+ "cc-name",
"cc-number",
"cc-csc",
"cc-exp",
"cc-exp-month",
"cc-exp-year",
]);
+ const CREDIT_CARD_HINT_PHRASES = [
+ "card number",
+ "credit card",
+ "cardholder",
+ "cvv",
+ "cvc",
+ ];
+ const TOTP_HINT_PHRASES = [
+ "otp",
+ "totp",
+ "2fa",
+ "two factor",
+ "one time code",
+ "verification code",
+ "authenticator code",
+ ];
+ const SECRET_HINT_PHRASES = [
+ "api key",
+ "apikey",
+ "access token",
+ "client secret",
+ "webhook secret",
+ "private key",
+ "bearer token",
+ "secret value",
+ ];
+
+ const normalizeHaystack = (...parts) => {
+ const joined = parts.filter(Boolean).join(" ").toLowerCase();
+ const normalized = joined.replace(/[^a-z0-9]+/g, " ").trim();
+ return normalized ? ` ${normalized} ` : "";
+ };
+
+ const haystackHasPhrase = (haystack, phrase) => {
+ if (!haystack) {
+ return false;
+ }
+ const needle = ` ${String(phrase)
+ .toLowerCase()
+ .replace(/[^a-z0-9]+/g, " ")
+ .trim()} `;
+ return haystack.includes(needle);
+ };
const isSecretField = (element) => {
if (!element) {
@@ -143,9 +187,32 @@
const autocomplete = String(
(element.getAttribute && element.getAttribute("autocomplete")) || "",
).toLowerCase();
- return autocomplete
- .split(/\s+/)
- .some((token) => SECRET_AUTOCOMPLETE_TOKENS.has(token));
+ if (
+ autocomplete
+ .split(/\s+/)
+ .some((token) => SECRET_AUTOCOMPLETE_TOKENS.has(token))
+ ) {
+ return true;
+ }
+ const haystack = normalizeHaystack(
+ element.id,
+ element.name,
+ element.getAttribute && element.getAttribute("name"),
+ element.getAttribute && element.getAttribute("aria-label"),
+ element.getAttribute && element.getAttribute("placeholder"),
+ element.getAttribute && element.getAttribute("title"),
+ );
+ return (
+ CREDIT_CARD_HINT_PHRASES.some((phrase) =>
+ haystackHasPhrase(haystack, phrase),
+ ) ||
+ TOTP_HINT_PHRASES.some((phrase) =>
+ haystackHasPhrase(haystack, phrase),
+ ) ||
+ SECRET_HINT_PHRASES.some((phrase) =>
+ haystackHasPhrase(haystack, phrase),
+ )
+ );
};
const IMPLICIT_INPUT_ROLES = {
diff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.py
index 468bf6ae1..f261f1caf 100644
--- a/skyvern/services/browser_recording/interpretation.py
+++ b/skyvern/services/browser_recording/interpretation.py
@@ -25,8 +25,10 @@ from skyvern.services.browser_recording.service import (
from skyvern.services.browser_recording.types import (
Action,
ActionBlockable,
+ ActionClick,
ActionInputText,
ActionKind,
+ CredentialKind,
ExfiltratedCdpEvent,
ExfiltratedConsoleEvent,
ExfiltratedEvent,
@@ -111,10 +113,17 @@ def _extra_field(block: OutputBlock, field_name: str, fallback: t.Any) -> t.Any:
return fallback
-def _credential_kind_for_action(action: Action) -> t.Literal["password", "totp", "credit_card"] | None:
- if not isinstance(action, ActionInputText):
+def _credential_kind_for_action(action: Action) -> CredentialKind | None:
+ if not isinstance(action, (ActionClick, ActionInputText)):
return None
- return credential_kind_for_target(action.target.input_type, action.target.autocomplete)
+ return credential_kind_for_target(
+ action.target.input_type,
+ action.target.autocomplete,
+ field_id=action.target.id,
+ accessible_name=action.target.accessible_name,
+ texts=action.target.texts,
+ tag_name=action.target.tag_name,
+ )
def _draft_step_from_block(
diff --git a/skyvern/services/browser_recording/redact.py b/skyvern/services/browser_recording/redact.py
index 33cb07956..2cda34e69 100644
--- a/skyvern/services/browser_recording/redact.py
+++ b/skyvern/services/browser_recording/redact.py
@@ -6,11 +6,10 @@ Keep the token lists identical to `isSecretField` in
from __future__ import annotations
+import re
import typing as t
-from skyvern.services.browser_recording.types import ExfiltratedConsoleEvent, ExfiltratedEvent
-
-CredentialKind = t.Literal["password", "totp", "credit_card"]
+from skyvern.services.browser_recording.types import CredentialKind, ExfiltratedConsoleEvent, ExfiltratedEvent
SECRET_INPUT_TYPES = frozenset({"password"})
SECRET_AUTOCOMPLETE_TOKENS = frozenset(
@@ -18,6 +17,7 @@ SECRET_AUTOCOMPLETE_TOKENS = frozenset(
"current-password",
"new-password",
"one-time-code",
+ "cc-name",
"cc-number",
"cc-csc",
"cc-exp",
@@ -28,6 +28,7 @@ SECRET_AUTOCOMPLETE_TOKENS = frozenset(
PASSWORD_AUTOCOMPLETE_TOKENS = frozenset({"current-password", "new-password"})
CREDIT_CARD_AUTOCOMPLETE_TOKENS = frozenset(
{
+ "cc-name",
"cc-number",
"cc-csc",
"cc-exp",
@@ -36,6 +37,45 @@ CREDIT_CARD_AUTOCOMPLETE_TOKENS = frozenset(
}
)
+# Phrase matchers run on a punctuation-normalized haystack of id / name / labels.
+CREDIT_CARD_HINT_PHRASES = (
+ "card number",
+ "credit card",
+ "cardholder",
+ "cvv",
+ "cvc",
+)
+TOTP_HINT_PHRASES = (
+ "otp",
+ "totp",
+ "2fa",
+ "two factor",
+ "one time code",
+ "verification code",
+ "authenticator code",
+)
+SECRET_HINT_PHRASES = (
+ "api key",
+ "apikey",
+ "access token",
+ "client secret",
+ "webhook secret",
+ "private key",
+ "bearer token",
+ "secret value",
+)
+MAGIC_LINK_HINT_PHRASES = (
+ "magic link",
+ "email me a link",
+ "send login link",
+ "send sign in link",
+ "passwordless",
+)
+
+_NON_ALNUM_RE = re.compile(r"[^a-z0-9]+")
+
+HaystackPart = str | None | list[str]
+
def _autocomplete_tokens(autocomplete: str | None) -> list[str]:
if not autocomplete:
@@ -43,20 +83,87 @@ def _autocomplete_tokens(autocomplete: str | None) -> list[str]:
return autocomplete.lower().split()
-def is_secret_field(input_type: str | None, autocomplete: str | None) -> bool:
- if (input_type or "").lower() in SECRET_INPUT_TYPES:
- return True
- return any(token in SECRET_AUTOCOMPLETE_TOKENS for token in _autocomplete_tokens(autocomplete))
-
-
-def credential_kind_for_target(input_type: str | None, autocomplete: str | None) -> CredentialKind | None:
+# `texts` comes from getElementText, which appends innerText/textContent for any element
+# that has children: for a <select> that is the option labels, for a container it is
+# arbitrary page copy. <input> is void, so only there is `texts` purely labelling. Anything
+# else is classified from its id and accessible name only, which is also what the page
+# script's own matcher reads.
+TEXT_IS_LABEL_TAGS = frozenset({"input"})
+
+
+def texts_are_labels(tag_name: str | None) -> bool:
+ return (tag_name or "").lower() in TEXT_IS_LABEL_TAGS
+
+
+def _normalized_haystack(*parts: HaystackPart) -> str:
+ chunks: list[str] = []
+ for part in parts:
+ if part is None:
+ continue
+ if isinstance(part, list):
+ chunks.extend(item for item in part if item)
+ elif part:
+ chunks.append(part)
+ if not chunks:
+ return ""
+ return " " + _NON_ALNUM_RE.sub(" ", " ".join(chunks).lower()).strip() + " "
+
+
+def _haystack_has_phrase(haystack: str, phrase: str) -> bool:
+ if not haystack:
+ return False
+ needle = " " + _NON_ALNUM_RE.sub(" ", phrase.lower()).strip() + " "
+ return needle in haystack
+
+
+def _haystack_has_any(haystack: str, phrases: t.Sequence[str]) -> bool:
+ return any(_haystack_has_phrase(haystack, phrase) for phrase in phrases)
+
+
+def is_secret_field(
+ input_type: str | None,
+ autocomplete: str | None,
+ *,
+ field_id: str | None = None,
+ accessible_name: str | None = None,
+ texts: list[str] | None = None,
+ tag_name: str | None = None,
+) -> bool:
+ kind = credential_kind_for_target(
+ input_type,
+ autocomplete,
+ field_id=field_id,
+ accessible_name=accessible_name,
+ texts=texts,
+ tag_name=tag_name,
+ )
+ return kind in {"password", "totp", "credit_card", "secret"}
+
+
+def credential_kind_for_target(
+ input_type: str | None,
+ autocomplete: str | None,
+ *,
+ field_id: str | None = None,
+ accessible_name: str | None = None,
+ texts: list[str] | None = None,
+ tag_name: str | None = None,
+) -> CredentialKind | None:
tokens = _autocomplete_tokens(autocomplete)
- if any(token in CREDIT_CARD_AUTOCOMPLETE_TOKENS for token in tokens):
+ haystack = _normalized_haystack(field_id, accessible_name, texts if texts_are_labels(tag_name) else None)
+
+ if any(token in CREDIT_CARD_AUTOCOMPLETE_TOKENS for token in tokens) or _haystack_has_any(
+ haystack, CREDIT_CARD_HINT_PHRASES
+ ):
return "credit_card"
- if "one-time-code" in tokens:
+ if "one-time-code" in tokens or _haystack_has_any(haystack, TOTP_HINT_PHRASES):
return "totp"
+ if _haystack_has_any(haystack, SECRET_HINT_PHRASES):
+ return "secret"
if (input_type or "").lower() == "password" or any(token in PASSWORD_AUTOCOMPLETE_TOKENS for token in tokens):
return "password"
+ if _haystack_has_any(haystack, MAGIC_LINK_HINT_PHRASES):
+ return "magic_link"
return None
@@ -74,7 +181,14 @@ def redact_console_event(event: ExfiltratedEvent) -> ExfiltratedEvent:
return event
target = event.params.target
- if not is_secret_field(target.inputType, target.autocomplete):
+ if not is_secret_field(
+ target.inputType,
+ target.autocomplete,
+ field_id=target.id,
+ accessible_name=target.accessibleName,
+ texts=target.text,
+ tag_name=target.tagName,
+ ):
return event
target.value = None
diff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.py
index fe3416327..f3ab55cd1 100644
--- a/skyvern/services/browser_recording/service.py
+++ b/skyvern/services/browser_recording/service.py
@@ -591,7 +591,14 @@ class Processor:
prompt_name = "recording-action-block-prompt-input-text"
if (
isinstance(action, ActionInputText)
- and is_secret_field(action.target.input_type, action.target.autocomplete)
+ and is_secret_field(
+ action.target.input_type,
+ action.target.autocomplete,
+ field_id=action.target.id,
+ accessible_name=action.target.accessible_name,
+ texts=action.target.texts,
+ tag_name=action.target.tag_name,
+ )
and action.input_value
):
action = action.model_copy(update={"input_value": ""})
diff --git a/skyvern/services/browser_recording/state_machines/input_text.py b/skyvern/services/browser_recording/state_machines/input_text.py
index b0b1a2780..41ce624c7 100644
--- a/skyvern/services/browser_recording/state_machines/input_text.py
+++ b/skyvern/services/browser_recording/state_machines/input_text.py
@@ -104,7 +104,14 @@ class StateMachineInputText(StateMachine):
LOG.debug("~ emitting input text action")
- secret = is_secret_field(self.target.inputType, self.target.autocomplete)
+ secret = is_secret_field(
+ self.target.inputType,
+ self.target.autocomplete,
+ field_id=self.target.id,
+ accessible_name=self.target.accessibleName,
+ texts=self.target.text,
+ tag_name=self.target.tagName,
+ )
input_value = event.params.target.value
if input_value is None and not secret:
diff --git a/skyvern/services/browser_recording/types.py b/skyvern/services/browser_recording/types.py
index 71de88ccd..ef86dba3f 100644
--- a/skyvern/services/browser_recording/types.py
+++ b/skyvern/services/browser_recording/types.py
@@ -78,6 +78,8 @@ Action = ActionClick | ActionHover | ActionInputText | ActionUrlChange | ActionW
ActionBlockable = ActionClick | ActionHover | ActionInputText
+CredentialKind = Literal["password", "totp", "credit_card", "secret", "magic_link"]
+
class ActionTarget(BaseModel):
class_name: str | None = None
@@ -134,7 +136,7 @@ class RecordingDraftStep(BaseModel):
# locally-captured artifacts (e.g. stream screenshots) with each step.
timestamp_start: float | None = None
timestamp_end: float | None = None
- credential_kind: Literal["password", "totp", "credit_card"] | None = None
+ credential_kind: CredentialKind | None = None
class RecordingInterpretationUpdate(BaseModel):
diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.py
index 7506f4db2..2694ef969 100644
--- a/tests/unit/services/test_browser_recording.py
+++ b/tests/unit/services/test_browser_recording.py
@@ -5,12 +5,15 @@ Just an example unit test for now. Will expand later.
import asyncio
import base64
import gzip
+import re
import time
import typing as t
import zlib
+from pathlib import Path
import pytest
+import skyvern
from skyvern.forge import app
from skyvern.forge.sdk.routes.streaming.channels.exfiltration import ExfiltratedEvent as StreamingExfiltratedEvent
from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (
@@ -19,6 +22,7 @@ from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (
from skyvern.forge.sdk.routes.streaming.channels.exfiltration import (
ExfiltrationChannel,
)
+from skyvern.services.browser_recording import redact as redact_module
from skyvern.services.browser_recording.interpretation import RecordingInterpretationSession
from skyvern.services.browser_recording.service import (
DUPLICATE_ACTION_WINDOW_MS,
@@ -236,12 +240,26 @@ def _empty_action_target() -> ActionTarget:
return ActionTarget(mouse=Mouse(xp=None, yp=None))
-def make_streaming_console_click(timestamp_ms: float) -> StreamingExfiltratedEvent:
+def make_streaming_console_click(
+ timestamp_ms: float,
+ *,
+ target_id: str = "button-1",
+ target_text: str = "Click me",
+ accessible_name: str | None = None,
+) -> StreamingExfiltratedEvent:
+ target: dict[str, t.Any] = {
+ "id": target_id,
+ "skyId": "sky-123",
+ "tagName": "BUTTON",
+ "text": [target_text],
+ }
+ if accessible_name is not None:
+ target["accessibleName"] = accessible_name
return StreamingExfiltratedEvent(
event_name="user_interaction",
params={
"type": "click",
- "target": {"id": "button-1", "skyId": "sky-123", "tagName": "BUTTON", "text": ["Click me"]},
+ "target": target,
"timestamp": timestamp_ms,
"url": "https://example.com",
"activeElement": {"tagName": "BUTTON"},
@@ -270,6 +288,7 @@ def make_streaming_console_input(
target_text: str = "Email",
input_type: str | None = None,
autocomplete: str | None = None,
+ accessible_name: str | None = None,
) -> list[StreamingExfiltratedEvent]:
target: dict[str, t.Any] = {
"id": target_id,
@@ -282,6 +301,8 @@ def make_streaming_console_input(
target["inputType"] = input_type
if autocomplete is not None:
target["autocomplete"] = autocomplete
+ if accessible_name is not None:
+ target["accessibleName"] = accessible_name
common = {
"target": target,
"timestamp": timestamp_ms,
@@ -921,9 +942,28 @@ def test_is_secret_field_and_credential_kind() -> None:
assert credential_kind_for_target("text", "current-password") == "password"
assert credential_kind_for_target("text", "one-time-code") == "totp"
assert credential_kind_for_target("text", "cc-number") == "credit_card"
+ assert credential_kind_for_target("text", "cc-name") == "credit_card"
assert credential_kind_for_target("email", None) is None
assert credential_kind_for_target("text", "off") is None
+ assert credential_kind_for_target("text", None, field_id="otp", texts=["Verification code"]) == "totp"
+ assert credential_kind_for_target("password", None, accessible_name="API Key") == "secret"
+ assert credential_kind_for_target("text", None, texts=["Card number"], tag_name="input") == "credit_card"
+
+ # `texts` carries innerText for elements with children, so it is only a label on <input>.
+ # A <select> whose options happen to contain a hint phrase must keep its recorded value.
+ assert credential_kind_for_target("text", None, texts=["Card number"], tag_name="select") is None
+ assert (
+ credential_kind_for_target("text", None, texts=["Reason: Verification code issue"], tag_name="select") is None
+ )
+ assert credential_kind_for_target(None, None, texts=["Two factor authentication"], tag_name="div") is None
+ # The label itself still classifies, whatever the tag.
+ assert credential_kind_for_target("text", None, accessible_name="Verification code", tag_name="select") == "totp"
+ assert credential_kind_for_target("password", None, accessible_name="Password") == "password"
+ assert credential_kind_for_target(None, None, accessible_name="Email me a magic link") == "magic_link"
+ assert is_secret_field("password", None, accessible_name="API Key") is True
+ assert is_secret_field(None, None, accessible_name="Email me a magic link") is False
+
def test_reify_strips_secret_values_from_console_events() -> None:
processor = Processor(PBS_ID, ORG_ID, WP_ID)
@@ -992,6 +1032,41 @@ def test_password_fill_still_emits_empty_input_text() -> None:
assert action.target.input_type == "password"
+def _js_string_literals(source: str, name: str) -> set[str]:
+ match = re.search(rf"const {name} = (?:new Set\()?\[(.*?)\]", source, re.DOTALL)
+ assert match, f"{name} not found in exfiltrate.js"
+ return set(re.findall(r'"([^"]*)"', match.group(1)))
+
+
+def test_secret_matcher_lists_match_between_js_and_python() -> None:
+ """The page script and the Python ingest must classify the same fields as secret.
+
+ A token added to one side only silently changes what gets captured, so parity is
+ asserted here rather than left to the docstring in redact.py.
+ """
+ js_source = (Path(skyvern.__file__).parent / "forge/sdk/routes/streaming/channels/js/exfiltrate.js").read_text()
+ matcher = re.search(r"const isSecretField = \(element\) => \{(.*?)\n \};", js_source, re.DOTALL)
+ assert matcher, "isSecretField not found in exfiltrate.js"
+ js_matcher_body = matcher.group(1)
+
+ # MAGIC_LINK_HINT_PHRASES is deliberately absent from the JS: "magic_link" is not a
+ # secret kind, so it labels a field without redacting it.
+ for name, python_value in (
+ ("SECRET_INPUT_TYPES", redact_module.SECRET_INPUT_TYPES),
+ ("SECRET_AUTOCOMPLETE_TOKENS", redact_module.SECRET_AUTOCOMPLETE_TOKENS),
+ ("CREDIT_CARD_HINT_PHRASES", redact_module.CREDIT_CARD_HINT_PHRASES),
+ ("TOTP_HINT_PHRASES", redact_module.TOTP_HINT_PHRASES),
+ ("SECRET_HINT_PHRASES", redact_module.SECRET_HINT_PHRASES),
+ ):
+ assert _js_string_literals(js_source, name) == set(python_value), (
+ f"{name} differs between exfiltrate.js and redact.py"
+ )
+ # Matching lists are worthless if the page script stops consulting one: deleting a
+ # clause from isSecretField while leaving its array declared would keep the parity
+ # assertion above green and silently stop redacting that family at source.
+ assert name in js_matcher_body, f"{name} is declared but never used by isSecretField"
+
+
def test_password_submitted_with_enter_still_emits_input_text() -> None:
"""Enter-to-submit never blurs before navigating, so the Enter keydown is the only emit signal.
@@ -1039,6 +1114,37 @@ def test_secret_character_keystrokes_still_redacted() -> None:
assert submit.params.target.value is None
+def test_phrase_only_secret_has_its_value_stripped_through_the_processor() -> None:
+ """A field classified *only* by its label must lose its value, not just get a kind.
+
+ The password/OTP cases are carried by input_type/autocomplete, so neither proves the
+ phrase matching this PR adds actually reaches redaction.
+ """
+ target = {
+ "id": "field-1",
+ "tagName": "INPUT",
+ "inputType": "text",
+ "autocomplete": None,
+ "accessibleName": "API Key",
+ "value": "sk_live_not_real",
+ }
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ events = processor.reify(
+ [
+ make_focus_event(target=target, timestamp=1000.0).model_dump(),
+ make_keydown_event(target=target, timestamp=1001.0, key="s").model_dump(),
+ make_blur_event(target=target, timestamp=1002.0).model_dump(),
+ ]
+ )
+ actions = processor.events_to_actions(events)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, ActionInputText)
+ assert action.input_value == ""
+ assert "sk_live_not_real" not in repr(action)
+
+
def test_non_secret_empty_value_still_skipped() -> None:
target = {
"id": "email",
@@ -1186,3 +1292,69 @@ async def test_otp_autocomplete_ingest_redacts_and_stamps_totp(
assert len(steps) == 1
assert steps[0].credential_kind == "totp"
assert "654321" not in (steps[0].navigation_goal or "")
+
+
+async def test_api_key_ingest_stamps_secret_kind(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:
+ raise RuntimeError("llm unavailable")
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)
+
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=lambda update: None,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ )
+ session.ingest_events(
+ make_streaming_console_input(
+ timestamp_ms=1000.0,
+ input_value="sk_live_secret",
+ target_id="api-key",
+ target_text="API Key",
+ input_type="password",
+ accessible_name="API Key",
+ )
+ )
+ steps = await session.flush()
+
+ assert len(steps) == 1
+ assert steps[0].credential_kind == "secret"
+ assert "sk_live_secret" not in (steps[0].navigation_goal or "")
+
+
+async def test_magic_link_click_stamps_kind(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:
+ raise RuntimeError("llm unavailable")
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)
+
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=lambda update: None,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ )
+ session.ingest_events(
+ [
+ make_streaming_console_click(
+ 1000.0,
+ target_text="Email me a magic link",
+ accessible_name="Email me a magic link",
+ )
+ ]
+ )
+ steps = await session.flush()
+
+ assert len(steps) == 1
+ assert steps[0].credential_kind == "magic_link"