Skip to content

Automate browser based workflows with AI

Save credentials from the live draft (#8307)

Cindy Lieda820580d75
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx
index 90f61c2b5..7e42de134 100644
--- a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx
@@ -4,12 +4,19 @@ import { cleanup, fireEvent, render, screen } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
-import { useRecordingStore } from "@/store/useRecordingStore";
+import {
+ useRecordingStore,
+ type RecordingDraftStep,
+} from "@/store/useRecordingStore";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
import { RecordingPanel } from "./RecordingPanel";
const mutateMock = vi.fn();
+const modalState = vi.hoisted(() => ({
+ overrideType: null as string | null,
+ defaultTestUrl: null as string | null,
+}));
vi.mock("@/routes/browserSessions/hooks/useProcessRecordingMutation", () => ({
useProcessRecordingMutation: () => ({
@@ -19,14 +26,65 @@ vi.mock("@/routes/browserSessions/hooks/useProcessRecordingMutation", () => ({
}),
}));
+vi.mock("@/routes/credentials/CredentialsModal", () => ({
+ CredentialsModal: ({
+ isOpen,
+ onCredentialCreated,
+ overrideType,
+ defaultTestUrl,
+ }: {
+ isOpen?: boolean;
+ onCredentialCreated?: (id: string, name?: string) => void;
+ overrideType?: string;
+ defaultTestUrl?: string;
+ }) => {
+ modalState.overrideType = overrideType ?? null;
+ modalState.defaultTestUrl = defaultTestUrl ?? null;
+ return isOpen ? (
+ <button
+ type="button"
+ data-testid="mock-create-credential"
+ onClick={() => onCredentialCreated?.("new-cred-1")}
+ >
+ create credential
+ </button>
+ ) : null;
+ },
+}));
+
const initialRecording = useRecordingStore.getState();
const initialPanel = useWorkflowPanelStore.getState();
const initialRecordedBlocks = useRecordedBlocksStore.getState();
+function inputDraft(
+ id: string,
+ opts: {
+ credential_kind?: RecordingDraftStep["credential_kind"];
+ url?: string;
+ title?: string;
+ } = {},
+): RecordingDraftStep {
+ return {
+ step_id: id,
+ action_kind: "input_text",
+ block_type: "action",
+ label: id,
+ title: opts.title ?? id,
+ status: "ready",
+ editable_fields: [],
+ parameters: [],
+ parameter_keys: [],
+ url: opts.url ?? "https://example.com/login",
+ credential_kind: opts.credential_kind,
+ };
+}
+
describe("RecordingPanel", () => {
beforeEach(() => {
HTMLElement.prototype.scrollIntoView = vi.fn();
mutateMock.mockReset();
+ modalState.overrideType = null;
+ modalState.defaultTestUrl = null;
useRecordingStore.setState(initialRecording, true);
useWorkflowPanelStore.setState(initialPanel, true);
useRecordedBlocksStore.setState(initialRecordedBlocks, true);
@@ -113,4 +171,72 @@ describe("RecordingPanel", () => {
vi.useRealTimers();
}
});
+
+ it("shows add and dismiss on a password draft, not on an email draft", () => {
+ useRecordingStore.setState({
+ draftSteps: [
+ inputDraft("email", { title: "Fill email" }),
+ inputDraft("pw", {
+ credential_kind: "password",
+ title: "Fill password",
+ }),
+ ],
+ });
+
+ render(<RecordingPanel browserSessionId="pbs_123" />);
+
+ expect(screen.getByText("Fill password")).toBeTruthy();
+ expect(screen.getByText("Fill email")).toBeTruthy();
+ expect(
+ screen.getByRole("button", { name: /add to credentials/i }),
+ ).toBeTruthy();
+ expect(screen.getByRole("button", { name: /^dismiss$/i })).toBeTruthy();
+ });
+
+ it("hides the credential prompt after dismiss", () => {
+ useRecordingStore.setState({
+ draftSteps: [
+ inputDraft("pw", {
+ credential_kind: "password",
+ title: "Fill password",
+ }),
+ ],
+ });
+
+ render(<RecordingPanel browserSessionId="pbs_123" />);
+ fireEvent.click(screen.getByRole("button", { name: /^dismiss$/i }));
+
+ expect(
+ screen.queryByRole("button", { name: /add to credentials/i }),
+ ).toBeNull();
+ expect(screen.getByText("Fill password")).toBeTruthy();
+ });
+
+ it("opens the password credential modal and dismisses the prompt after create", () => {
+ useRecordingStore.setState({
+ draftSteps: [
+ inputDraft("pw", {
+ credential_kind: "password",
+ title: "Fill password",
+ url: "https://example.com/login",
+ }),
+ ],
+ });
+
+ render(<RecordingPanel browserSessionId="pbs_123" />);
+ fireEvent.click(
+ screen.getByRole("button", { name: /add to credentials/i }),
+ );
+
+ expect(modalState.overrideType).toBe("password");
+ expect(modalState.defaultTestUrl).toBe("https://example.com/login");
+ expect(screen.getByTestId("mock-create-credential")).toBeTruthy();
+
+ fireEvent.click(screen.getByTestId("mock-create-credential"));
+
+ expect(
+ screen.queryByRole("button", { name: /add to credentials/i }),
+ ).toBeNull();
+ expect(screen.queryByTestId("mock-create-credential")).toBeNull();
+ });
});
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx
index ea6970676..7f4295b52 100644
--- a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx
+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx
@@ -22,6 +22,11 @@ import {
} from "@/components/ui/dialog";
import { useRecordingElapsedSeconds } from "@/hooks/useRecordingElapsedSeconds";
import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";
+import { CredentialsModal } from "@/routes/credentials/CredentialsModal";
+import {
+ CredentialModalTypes,
+ type CredentialModalType,
+} from "@/routes/credentials/useCredentialModalState";
import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";
import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
import {
@@ -132,6 +137,14 @@ function StepScreenshot({ screenshot }: { screenshot: RecordingScreenshot }) {
);
}
+function credentialModalTypeForKind(
+ kind: NonNullable<RecordingDraftStep["credential_kind"]>,
+): CredentialModalType {
+ return kind === "credit_card"
+ ? CredentialModalTypes.CREDIT_CARD
+ : CredentialModalTypes.PASSWORD;
+}
+
function DraftStepCard({
step,
index,
@@ -139,6 +152,9 @@ function DraftStepCard({
screenshot,
onDelete,
onRename,
+ showCredentialPrompt,
+ onAddCredentials,
+ onDismissCredential,
}: {
step: RecordingDraftStep;
index: number;
@@ -146,6 +162,9 @@ function DraftStepCard({
screenshot: RecordingScreenshot | null;
onDelete: () => void;
onRename: (value: string) => void;
+ showCredentialPrompt: boolean;
+ onAddCredentials: () => void;
+ onDismissCredential: () => void;
}) {
const beginDraftEdit = useRecordingStore((state) => state.beginDraftEdit);
const endDraftEdit = useRecordingStore((state) => state.endDraftEdit);
@@ -267,6 +286,27 @@ function DraftStepCard({
<StepScreenshot screenshot={screenshot} />
</div>
)}
+ {showCredentialPrompt && (
+ <div className="flex flex-wrap items-center gap-2 pl-8">
+ <Button
+ type="button"
+ size="sm"
+ className="h-7"
+ onClick={onAddCredentials}
+ >
+ Add to credentials
+ </Button>
+ <Button
+ type="button"
+ variant="ghost"
+ size="sm"
+ className="h-7"
+ onClick={onDismissCredential}
+ >
+ Dismiss
+ </Button>
+ </div>
+ )}
</div>
);
}
@@ -335,6 +375,12 @@ function RecordingPanel({ browserSessionId }: Props) {
// the mutation guard throw.
const browserSessionMissing = !browserSessionId;
const [confirmDiscardOpen, setConfirmDiscardOpen] = useState(false);
+ const [credentialModal, setCredentialModal] = useState<{
+ type: CredentialModalType;
+ testUrl: string | null;
+ url: string | null;
+ stepId: string;
+ } | null>(null);
const feedEndRef = useRef<HTMLDivElement | null>(null);
const committedRef = useRef(false);
@@ -345,6 +391,7 @@ function RecordingPanel({ browserSessionId }: Props) {
draftSteps,
deletedStepIds,
stepPatches,
+ dismissedCredentialStepIds,
screenshots,
sessionRevision,
optimisticSteps: rawOptimisticSteps,
@@ -360,6 +407,7 @@ function RecordingPanel({ browserSessionId }: Props) {
draftSteps: state.draftSteps,
deletedStepIds: state.deletedStepIds,
stepPatches: state.stepPatches,
+ dismissedCredentialStepIds: state.dismissedCredentialStepIds,
screenshots: state.screenshots,
sessionRevision: state.sessionRevision,
optimisticSteps: state.optimisticSteps,
@@ -553,6 +601,26 @@ function RecordingPanel({ browserSessionId }: Props) {
index={index}
baselineMs={baselineMs}
screenshot={findScreenshotForStep(step, screenshots)}
+ showCredentialPrompt={
+ Boolean(step.credential_kind) &&
+ !dismissedCredentialStepIds.includes(step.step_id)
+ }
+ onAddCredentials={() => {
+ if (!step.credential_kind) {
+ return;
+ }
+ setCredentialModal({
+ type: credentialModalTypeForKind(step.credential_kind),
+ testUrl: step.url ?? null,
+ url: step.url ?? null,
+ stepId: step.step_id,
+ });
+ }}
+ onDismissCredential={() =>
+ useRecordingStore
+ .getState()
+ .dismissCredentialPrompt(step.step_id)
+ }
onDelete={() =>
useRecordingStore.getState().deleteDraftStep(step.step_id)
}
@@ -671,6 +739,27 @@ function RecordingPanel({ browserSessionId }: Props) {
</DialogContent>
</Dialog>
)}
+ {credentialModal ? (
+ <CredentialsModal
+ isOpen
+ overrideType={credentialModal.type}
+ defaultTestUrl={credentialModal.testUrl ?? undefined}
+ onOpenChange={(open) => {
+ if (!open) {
+ setCredentialModal(null);
+ }
+ }}
+ onCredentialCreated={() => {
+ const store = useRecordingStore.getState();
+ if (credentialModal.url) {
+ store.dismissCredentialPromptsForUrl(credentialModal.url);
+ } else {
+ store.dismissCredentialPrompt(credentialModal.stepId);
+ }
+ setCredentialModal(null);
+ }}
+ />
+ ) : null}
</div>
);
}
diff --git a/skyvern-frontend/src/store/useRecordingStore.test.ts b/skyvern-frontend/src/store/useRecordingStore.test.ts
index 855001f40..372f86a79 100644
--- a/skyvern-frontend/src/store/useRecordingStore.test.ts
+++ b/skyvern-frontend/src/store/useRecordingStore.test.ts
@@ -373,3 +373,51 @@ describe("applyInterpretationUpdate: snapshot vs delta", () => {
expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-0"]);
});
});
+
+describe("dismissCredentialPrompt", () => {
+ it("hides the prompt for one step and clears on a new recording", () => {
+ store().setIsRecording(true);
+ store().applyInterpretationUpdate(
+ update("s1", 1, [
+ { ...draft("pw", "input_text"), credential_kind: "password" },
+ { ...draft("otp", "input_text"), credential_kind: "totp" },
+ ]),
+ );
+
+ store().dismissCredentialPrompt("pw");
+ expect(store().dismissedCredentialStepIds).toEqual(["pw"]);
+
+ store().dismissCredentialPrompt("pw");
+ expect(store().dismissedCredentialStepIds).toEqual(["pw"]);
+
+ store().reset();
+ store().setIsRecording(true);
+ expect(store().dismissedCredentialStepIds).toEqual([]);
+ });
+
+ it("dismisses every secret step on the same url", () => {
+ store().setIsRecording(true);
+ store().applyInterpretationUpdate(
+ update("s1", 1, [
+ {
+ ...draft("pw", "input_text"),
+ credential_kind: "password",
+ url: "https://example.com/login",
+ },
+ {
+ ...draft("otp", "input_text"),
+ credential_kind: "totp",
+ url: "https://example.com/login",
+ },
+ {
+ ...draft("other", "input_text"),
+ credential_kind: "password",
+ url: "https://other.example/login",
+ },
+ ]),
+ );
+
+ store().dismissCredentialPromptsForUrl("https://example.com/login");
+ expect(store().dismissedCredentialStepIds).toEqual(["pw", "otp"]);
+ });
+});
diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.ts
index 23e504e97..f0ca1e33d 100644
--- a/skyvern-frontend/src/store/useRecordingStore.ts
+++ b/skyvern-frontend/src/store/useRecordingStore.ts
@@ -127,6 +127,7 @@ export interface RecordingDraftStep {
parameter_keys: Array<string>;
timestamp_start?: number | null;
timestamp_end?: number | null;
+ credential_kind?: "password" | "totp" | "credit_card" | null;
}
export interface RecordingInterpretationUpdate {
@@ -285,6 +286,7 @@ interface RecordingStore {
*/
deletedStepIds: Array<string>;
stepPatches: Record<string, RecordingDraftStepPatch>;
+ dismissedCredentialStepIds: Array<string>;
/**
* Nested count of in-progress live draft title edits. While > 0, capture is
* paused alongside manualCapturePaused.
@@ -313,6 +315,8 @@ interface RecordingStore {
addOptimisticStep: (step: OptimisticStep) => void;
deleteDraftStep: (stepId: string) => void;
patchDraftStep: (stepId: string, patch: RecordingDraftStepPatch) => void;
+ dismissCredentialPrompt: (stepId: string) => void;
+ dismissCredentialPromptsForUrl: (url: string | null | undefined) => void;
beginDraftEdit: () => void;
endDraftEdit: () => void;
setManualCapturePaused: (paused: boolean) => void;
@@ -511,6 +515,7 @@ function emptyRecordingState() {
interpretationFinalized: false,
deletedStepIds: [] as Array<string>,
stepPatches: {} as Record<string, RecordingDraftStepPatch>,
+ dismissedCredentialStepIds: [] as Array<string>,
draftEditDepth: 0,
manualCapturePaused: false,
screenshots: [] as Array<RecordingScreenshot>,
@@ -537,6 +542,7 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
interpretationFinalized: false,
deletedStepIds: [],
stepPatches: {},
+ dismissedCredentialStepIds: [],
draftEditDepth: 0,
manualCapturePaused: false,
screenshots: [],
@@ -595,7 +601,13 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
...(clearOptimistic ? { optimisticSteps: [] } : {}),
interpretationPending: update.pending,
interpretationFinalized: update.finalized,
- ...(sessionChanged ? { deletedStepIds: [], stepPatches: {} } : {}),
+ ...(sessionChanged
+ ? {
+ deletedStepIds: [],
+ stepPatches: {},
+ dismissedCredentialStepIds: [],
+ }
+ : {}),
});
},
@@ -635,6 +647,34 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({
});
},
+ dismissCredentialPrompt: (stepId) => {
+ const state = get();
+ if (state.dismissedCredentialStepIds.includes(stepId)) {
+ return;
+ }
+ set({
+ dismissedCredentialStepIds: [...state.dismissedCredentialStepIds, stepId],
+ });
+ },
+
+ dismissCredentialPromptsForUrl: (url) => {
+ if (!url) {
+ return;
+ }
+ const state = get();
+ const dismissed = new Set(state.dismissedCredentialStepIds);
+ for (const step of state.draftSteps) {
+ if (!step.credential_kind || step.url !== url) {
+ continue;
+ }
+ dismissed.add(step.step_id);
+ }
+ if (dismissed.size === state.dismissedCredentialStepIds.length) {
+ return;
+ }
+ set({ dismissedCredentialStepIds: Array.from(dismissed) });
+ },
+
beginDraftEdit: () => {
set({ draftEditDepth: get().draftEditDepth + 1 });
},
diff --git a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
index ca4a04059..692e3a81d 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
+++ b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
@@ -120,6 +120,34 @@
return isUniqueSelector(selector) ? selector : null;
};
+ const SECRET_INPUT_TYPES = new Set(["password"]);
+ const SECRET_AUTOCOMPLETE_TOKENS = new Set([
+ "current-password",
+ "new-password",
+ "one-time-code",
+ "cc-number",
+ "cc-csc",
+ "cc-exp",
+ "cc-exp-month",
+ "cc-exp-year",
+ ]);
+
+ const isSecretField = (element) => {
+ if (!element) {
+ return false;
+ }
+ const inputType = String(element.type || "").toLowerCase();
+ if (SECRET_INPUT_TYPES.has(inputType)) {
+ return true;
+ }
+ const autocomplete = String(
+ (element.getAttribute && element.getAttribute("autocomplete")) || "",
+ ).toLowerCase();
+ return autocomplete
+ .split(/\s+/)
+ .some((token) => SECRET_AUTOCOMPLETE_TOKENS.has(token));
+ };
+
const IMPLICIT_INPUT_ROLES = {
button: "button",
submit: "button",
@@ -346,7 +374,7 @@
eventType,
id: e.target?.id,
className: e.target?.className,
- value: e.target?.value,
+ value: isSecretField(e.target) ? null : e.target?.value,
text: getElementText(e.target),
labels: getAssociatedLabels(e.target),
skyId: e.target?.dataset?.skyId,
@@ -368,6 +396,12 @@
const classText = String(
e.target.classList?.value ?? e.target.getAttribute("class") ?? "",
);
+ const secretField = isSecretField(e.target);
+ const autocomplete = e.target?.getAttribute?.("autocomplete") || null;
+ // Named keys ("Enter", "Tab") survive: the input-text state machine emits on Enter,
+ // which is the only signal an Enter-submitted login produces before it navigates away.
+ const redactKeystroke =
+ secretField && typeof e.key === "string" && e.key.length === 1;
const eventData = {
url: window.location.href,
@@ -379,7 +413,7 @@
isHtml: e.target instanceof HTMLElement,
isSvg: e.target instanceof SVGElement,
className: classText,
- value: e.target?.value,
+ value: secretField ? null : e.target?.value,
text: getElementText(e.target),
labels: getAssociatedLabels(e.target),
skyId: e.target?.dataset?.skyId,
@@ -399,10 +433,13 @@
e.target?.tagName === "BUTTON"
? e.target?.type || null
: null,
+ autocomplete,
},
- inputValue: ["input", "focus", "blur"].includes(eventType)
- ? e.target?.value
- : undefined,
+ inputValue: secretField
+ ? null
+ : ["input", "focus", "blur"].includes(eventType)
+ ? e.target?.value
+ : undefined,
mousePosition: {
xa: Number.isFinite(e.clientX) ? e.clientX : null,
ya: Number.isFinite(e.clientY) ? e.clientY : null,
@@ -415,8 +452,8 @@
? e.clientY / window.innerHeight
: null,
},
- key: e.key,
- code: e.code,
+ key: redactKeystroke ? null : e.key,
+ code: redactKeystroke ? null : e.code,
activeElement: {
tagName: document.activeElement?.tagName,
id: document.activeElement?.id,
diff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.py
index d1bb98962..468bf6ae1 100644
--- a/skyvern/services/browser_recording/interpretation.py
+++ b/skyvern/services/browser_recording/interpretation.py
@@ -16,6 +16,7 @@ from skyvern.client.types.workflow_definition_yaml_blocks_item import (
from skyvern.config import settings
from skyvern.forge.sdk.routes.streaming.channels import exfiltration as streaming_exfiltration
from skyvern.forge.sdk.routes.streaming.channels.exfiltration import ExfiltratedEventSource
+from skyvern.services.browser_recording.redact import credential_kind_for_target, redact_console_event
from skyvern.services.browser_recording.service import (
Processor,
deterministic_input_text_parameter_key,
@@ -77,7 +78,7 @@ def streaming_events_to_recording_events(
if source == "cdp":
reified_events.append(ExfiltratedCdpEvent(**payload))
elif source == "console":
- reified_events.append(ExfiltratedConsoleEvent(**payload))
+ reified_events.append(redact_console_event(ExfiltratedConsoleEvent(**payload)))
else:
LOG.debug("Skipping recording interpretation event with unsupported source", source=source)
except Exception:
@@ -110,6 +111,12 @@ def _extra_field(block: OutputBlock, field_name: str, fallback: t.Any) -> t.Any:
return fallback
+def _credential_kind_for_action(action: Action) -> t.Literal["password", "totp", "credit_card"] | None:
+ if not isinstance(action, ActionInputText):
+ return None
+ return credential_kind_for_target(action.target.input_type, action.target.autocomplete)
+
+
def _draft_step_from_block(
*,
browser_session_id: str,
@@ -138,6 +145,7 @@ def _draft_step_from_block(
parameter_keys=block.parameter_keys or [],
timestamp_start=action.timestamp_start,
timestamp_end=action.timestamp_end,
+ credential_kind=_credential_kind_for_action(action),
)
if isinstance(block, WorkflowDefinitionYamlBlocksItem_GotoUrl):
@@ -223,6 +231,7 @@ def _placeholder_step_from_action(
label=normalize_recording_block_label(f"{action.kind.value}_{text}", fallback=action.kind.value),
title=title,
navigation_goal=navigation_goal,
+ url=action.url,
status=RecordingDraftStepStatus.INTERPRETING,
editable_fields=[
RecordingDraftStepEditableField.LABEL,
@@ -233,6 +242,7 @@ def _placeholder_step_from_action(
parameter_keys=parameter_keys,
timestamp_start=action.timestamp_start,
timestamp_end=action.timestamp_end,
+ credential_kind=_credential_kind_for_action(action),
)
@@ -508,6 +518,10 @@ class RecordingInterpretationSession:
step.navigation_goal = enriched.navigation_goal
step.parameters = enriched.parameters
step.parameter_keys = enriched.parameter_keys
+ if enriched.credential_kind:
+ step.credential_kind = enriched.credential_kind
+ if enriched.url:
+ step.url = enriched.url
step.status = RecordingDraftStepStatus.READY
self._emit(changed_steps=[step])
diff --git a/skyvern/services/browser_recording/redact.py b/skyvern/services/browser_recording/redact.py
new file mode 100644
index 000000000..33cb07956
--- /dev/null
+++ b/skyvern/services/browser_recording/redact.py
@@ -0,0 +1,87 @@
+"""Secret-field detection for record-browser capture and live drafts.
+
+Keep the token lists identical to `isSecretField` in
+`skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js`.
+"""
+
+from __future__ import annotations
+
+import typing as t
+
+from skyvern.services.browser_recording.types import ExfiltratedConsoleEvent, ExfiltratedEvent
+
+CredentialKind = t.Literal["password", "totp", "credit_card"]
+
+SECRET_INPUT_TYPES = frozenset({"password"})
+SECRET_AUTOCOMPLETE_TOKENS = frozenset(
+ {
+ "current-password",
+ "new-password",
+ "one-time-code",
+ "cc-number",
+ "cc-csc",
+ "cc-exp",
+ "cc-exp-month",
+ "cc-exp-year",
+ }
+)
+PASSWORD_AUTOCOMPLETE_TOKENS = frozenset({"current-password", "new-password"})
+CREDIT_CARD_AUTOCOMPLETE_TOKENS = frozenset(
+ {
+ "cc-number",
+ "cc-csc",
+ "cc-exp",
+ "cc-exp-month",
+ "cc-exp-year",
+ }
+)
+
+
+def _autocomplete_tokens(autocomplete: str | None) -> list[str]:
+ if not autocomplete:
+ return []
+ return autocomplete.lower().split()
+
+
+def is_secret_field(input_type: str | None, autocomplete: str | None) -> bool:
+ if (input_type or "").lower() in SECRET_INPUT_TYPES:
+ return True
+ return any(token in SECRET_AUTOCOMPLETE_TOKENS for token in _autocomplete_tokens(autocomplete))
+
+
+def credential_kind_for_target(input_type: str | None, autocomplete: str | None) -> CredentialKind | None:
+ tokens = _autocomplete_tokens(autocomplete)
+ if any(token in CREDIT_CARD_AUTOCOMPLETE_TOKENS for token in tokens):
+ return "credit_card"
+ if "one-time-code" in tokens:
+ return "totp"
+ if (input_type or "").lower() == "password" or any(token in PASSWORD_AUTOCOMPLETE_TOKENS for token in tokens):
+ return "password"
+ return None
+
+
+def is_character_key(key: str | None) -> bool:
+ """Whether a KeyboardEvent.key carries typed content rather than naming a key.
+
+ Printable keys are a single character ("a", "1"); named keys are longer ("Enter", "Tab").
+ Only the former can reconstruct a secret, so only the former is dropped.
+ """
+ return key is not None and len(key) == 1
+
+
+def redact_console_event(event: ExfiltratedEvent) -> ExfiltratedEvent:
+ if not isinstance(event, ExfiltratedConsoleEvent):
+ return event
+
+ target = event.params.target
+ if not is_secret_field(target.inputType, target.autocomplete):
+ return event
+
+ target.value = None
+ event.params.inputValue = None
+ # Named keys survive: StateMachineInputText emits on Enter, which is the only signal a
+ # login submitted with Enter (never blurring before navigation) ever produces.
+ if is_character_key(event.params.key):
+ event.params.key = None
+ event.params.code = None
+ return event
diff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.py
index 4f32b0756..fe3416327 100644
--- a/skyvern/services/browser_recording/service.py
+++ b/skyvern/services/browser_recording/service.py
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.services.browser_recording.code_first import actions_to_code_first_blocks
+from skyvern.services.browser_recording.redact import is_secret_field, redact_console_event
from skyvern.services.browser_recording.types import (
Action,
ActionBlockable,
@@ -330,7 +331,7 @@ class Processor:
else:
LOG.error(f"{self.class_name} Unknown event source: {event.get('source')}", **self.identity)
continue
- reified_events.append(reified_event)
+ reified_events.append(redact_console_event(reified_event))
return reified_events
@@ -588,6 +589,12 @@ class Processor:
if action.kind == ActionKind.INPUT_TEXT:
prompt_name = "recording-action-block-prompt-input-text"
+ if (
+ isinstance(action, ActionInputText)
+ and is_secret_field(action.target.input_type, action.target.autocomplete)
+ and action.input_value
+ ):
+ action = action.model_copy(update={"input_value": ""})
else:
prompt_name = "recording-action-block-prompt"
diff --git a/skyvern/services/browser_recording/state_machines/input_text.py b/skyvern/services/browser_recording/state_machines/input_text.py
index c1b799a16..b0b1a2780 100644
--- a/skyvern/services/browser_recording/state_machines/input_text.py
+++ b/skyvern/services/browser_recording/state_machines/input_text.py
@@ -2,6 +2,7 @@ import typing as t
import structlog
+from skyvern.services.browser_recording.redact import is_secret_field
from skyvern.services.browser_recording.types import (
Action,
ActionInputText,
@@ -101,11 +102,12 @@ class StateMachineInputText(StateMachine):
xp = (self.mouse.xp or -1) if self.mouse else None
yp = (self.mouse.yp or -1) if self.mouse else None
- LOG.debug("~ emitting input text action", exfiltrated_event=event)
+ LOG.debug("~ emitting input text action")
+ secret = is_secret_field(self.target.inputType, self.target.autocomplete)
input_value = event.params.target.value
- if input_value is None:
+ if input_value is None and not secret:
LOG.debug("~ cannot emit, missing input value; resetting")
self.reset()
@@ -122,6 +124,7 @@ class StateMachineInputText(StateMachine):
role=self.target.role,
accessible_name=self.target.accessibleName,
input_type=self.target.inputType,
+ autocomplete=self.target.autocomplete,
)
action = ActionInputText(
@@ -130,7 +133,7 @@ class StateMachineInputText(StateMachine):
timestamp_start=self.timestamp_start,
timestamp_end=event.params.timestamp,
url=event.params.url,
- input_value=str(input_value),
+ input_value="" if secret else str(input_value),
)
self.reset()
diff --git a/skyvern/services/browser_recording/types.py b/skyvern/services/browser_recording/types.py
index adf26958d..71de88ccd 100644
--- a/skyvern/services/browser_recording/types.py
+++ b/skyvern/services/browser_recording/types.py
@@ -92,6 +92,7 @@ class ActionTarget(BaseModel):
role: str | None = None
accessible_name: str | None = None
input_type: str | None = None
+ autocomplete: str | None = None
class Mouse(BaseModel):
@@ -133,6 +134,7 @@ class RecordingDraftStep(BaseModel):
# locally-captured artifacts (e.g. stream screenshots) with each step.
timestamp_start: float | None = None
timestamp_end: float | None = None
+ credential_kind: Literal["password", "totp", "credit_card"] | None = None
class RecordingInterpretationUpdate(BaseModel):
@@ -195,6 +197,7 @@ class EventTarget(BaseModel):
role: str | None = None
accessibleName: str | None = None
inputType: str | None = None
+ autocomplete: str | None = None
class MousePosition(BaseModel):
diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.py
index 2515a3b71..7506f4db2 100644
--- a/tests/unit/services/test_browser_recording.py
+++ b/tests/unit/services/test_browser_recording.py
@@ -268,14 +268,20 @@ def make_streaming_console_input(
input_value: str,
target_id: str = "email",
target_text: str = "Email",
+ input_type: str | None = None,
+ autocomplete: str | None = None,
) -> list[StreamingExfiltratedEvent]:
- target = {
+ target: dict[str, t.Any] = {
"id": target_id,
"skyId": "sky-email",
"tagName": "INPUT",
"text": [target_text],
"value": input_value,
}
+ if input_type is not None:
+ target["inputType"] = input_type
+ if autocomplete is not None:
+ target["autocomplete"] = autocomplete
common = {
"target": target,
"timestamp": timestamp_ms,
@@ -884,3 +890,299 @@ def test_decompress_returns_bytes_for_valid_payload() -> None:
payload = base64.b64encode(gzip.compress(raw)).decode("ascii")
assert processor.decompress(payload) == raw
+
+
+def make_keydown_event(target: dict[str, t.Any], timestamp: float, key: str = "a") -> ExfiltratedConsoleEvent:
+ return make_console_event(
+ params={"type": "keydown", "key": key, "target": target, "timestamp": timestamp},
+ timestamp=timestamp,
+ )
+
+
+def make_blur_event(target: dict[str, t.Any], timestamp: float) -> ExfiltratedConsoleEvent:
+ return make_console_event(
+ params={"type": "blur", "target": target, "timestamp": timestamp},
+ timestamp=timestamp,
+ )
+
+
+def test_is_secret_field_and_credential_kind() -> None:
+ from skyvern.services.browser_recording.redact import credential_kind_for_target, is_secret_field
+
+ assert is_secret_field("password", None) is True
+ assert is_secret_field("text", "one-time-code") is True
+ assert is_secret_field("text", "cc-number") is True
+ assert is_secret_field("text", "username current-password") is True
+ assert is_secret_field("email", None) is False
+ assert is_secret_field("text", "off") is False
+ assert is_secret_field("text", "username") is False
+
+ assert credential_kind_for_target("password", None) == "password"
+ assert credential_kind_for_target("text", "current-password") == "password"
+ assert credential_kind_for_target("text", "one-time-code") == "totp"
+ assert credential_kind_for_target("text", "cc-number") == "credit_card"
+ assert credential_kind_for_target("email", None) is None
+ assert credential_kind_for_target("text", "off") is None
+
+
+def test_reify_strips_secret_values_from_console_events() -> None:
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ events = processor.reify(
+ [
+ {
+ "kind": "exfiltrated-event",
+ "event_name": "user_interaction",
+ "source": "console",
+ "timestamp": 1.0,
+ "params": {
+ "type": "keydown",
+ "key": "h",
+ "code": "KeyH",
+ "inputValue": "hunter2",
+ "url": "https://example.com/login",
+ "timestamp": 1000.0,
+ "target": {
+ "tagName": "INPUT",
+ "id": "password",
+ "skyId": "sky-pw",
+ "inputType": "password",
+ "value": "hunter2",
+ "text": ["Password"],
+ },
+ "activeElement": {"tagName": "INPUT"},
+ "window": {"height": 800, "width": 1200, "scrollX": 0, "scrollY": 0},
+ "mousePosition": {"xp": 0.5, "yp": 0.5},
+ },
+ }
+ ]
+ )
+
+ assert len(events) == 1
+ event = events[0]
+ assert isinstance(event, ExfiltratedConsoleEvent)
+ assert event.params.target.value is None
+ assert event.params.inputValue is None
+ assert event.params.key is None
+ assert event.params.code is None
+ assert event.params.target.inputType == "password"
+
+
+def test_password_fill_still_emits_empty_input_text() -> None:
+ target = {
+ "id": "password",
+ "skyId": "sky-pw",
+ "tagName": "INPUT",
+ "text": ["Password"],
+ "inputType": "password",
+ "value": None,
+ }
+ events = [
+ make_focus_event(target=target, timestamp=1000.0),
+ make_keydown_event(target=target, timestamp=1001.0),
+ make_blur_event(target=target, timestamp=1002.0),
+ ]
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ actions = processor.events_to_actions(events)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, ActionInputText)
+ assert action.input_value == ""
+ assert action.target.input_type == "password"
+
+
+def test_password_submitted_with_enter_still_emits_input_text() -> None:
+ """Enter-to-submit never blurs before navigating, so the Enter keydown is the only emit signal.
+
+ Redacting it along with the character keystrokes silently drops the whole password step.
+ """
+ target = {
+ "id": "password",
+ "skyId": "sky-pw",
+ "tagName": "INPUT",
+ "text": ["Password"],
+ "inputType": "password",
+ "value": None,
+ }
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ # Enter through reify(), not events_to_actions(): ingest redaction runs there, so a test
+ # below it would pass even with the Enter keydown stripped.
+ events = processor.reify(
+ [
+ make_focus_event(target=target, timestamp=1000.0).model_dump(),
+ make_keydown_event(target=target, timestamp=1001.0, key="s").model_dump(),
+ make_keydown_event(target=target, timestamp=1002.0, key="Enter").model_dump(),
+ ]
+ )
+ actions = processor.events_to_actions(events)
+
+ assert len(actions) == 1
+ action = actions[0]
+ assert isinstance(action, ActionInputText)
+ assert action.input_value == ""
+ assert action.target.input_type == "password"
+
+
+def test_secret_character_keystrokes_still_redacted() -> None:
+ from skyvern.services.browser_recording.redact import redact_console_event
+
+ target = {"id": "password", "tagName": "INPUT", "inputType": "password", "value": "hunter2"}
+
+ typed = redact_console_event(make_keydown_event(target=target, timestamp=1.0, key="s"))
+ assert typed.params.key is None
+ assert typed.params.code is None
+ assert typed.params.target.value is None
+
+ submit = redact_console_event(make_keydown_event(target=target, timestamp=2.0, key="Enter"))
+ assert submit.params.key == "Enter"
+ assert submit.params.target.value is None
+
+
+def test_non_secret_empty_value_still_skipped() -> None:
+ target = {
+ "id": "email",
+ "skyId": "sky-email",
+ "tagName": "INPUT",
+ "text": ["Email"],
+ "inputType": "email",
+ "value": None,
+ }
+ events = [
+ make_focus_event(target=target, timestamp=1000.0),
+ make_keydown_event(target=target, timestamp=1001.0),
+ make_blur_event(target=target, timestamp=1002.0),
+ ]
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ assert processor.events_to_actions(events) == []
+
+
+async def test_password_ingest_emits_redacted_draft_with_credential_kind(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:
+ raise RuntimeError("llm unavailable")
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)
+
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=lambda update: None,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ )
+
+ session.ingest_events(
+ make_streaming_console_input(
+ timestamp_ms=1000.0,
+ input_value="hunter2",
+ target_id="password",
+ target_text="Password",
+ input_type="password",
+ )
+ )
+ steps = await session.flush()
+
+ assert len(steps) == 1
+ assert steps[0].credential_kind == "password"
+ assert "hunter2" not in (steps[0].title or "")
+ assert "hunter2" not in (steps[0].navigation_goal or "")
+ stored = session.events
+ for event in stored:
+ if isinstance(event, ExfiltratedConsoleEvent):
+ assert event.params.target.value != "hunter2"
+ assert event.params.inputValue != "hunter2"
+
+
+async def test_create_action_block_prompt_omits_secret_keeps_email(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ import skyvern.services.browser_recording.service as svc
+
+ captured: dict[str, str] = {}
+
+ async def fake_llm(*, prompt: str, prompt_name: str, organization_id: str) -> dict[str, t.Any]:
+ captured[prompt_name] = prompt
+ return {"block_label": "fill", "title": "Fill", "prompt": "Fill the field."}
+
+ monkeypatch.setattr(svc, "_recording_enrichment_llm_handler", lambda: fake_llm)
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+
+ await processor.create_action_block(
+ ActionInputText(
+ kind=ActionKind.INPUT_TEXT,
+ target=ActionTarget(
+ id="password",
+ sky_id="sky-pw",
+ tag_name="INPUT",
+ texts=["Password"],
+ input_type="password",
+ mouse=Mouse(xp=0.5, yp=0.5),
+ ),
+ timestamp_start=1000.0,
+ timestamp_end=1001.0,
+ url="https://example.com/login",
+ input_value="hunter2",
+ )
+ )
+ assert "hunter2" not in captured["recording-action-block-prompt-input-text"]
+
+ captured.clear()
+ await processor.create_action_block(
+ ActionInputText(
+ kind=ActionKind.INPUT_TEXT,
+ target=ActionTarget(
+ id="email",
+ sky_id="sky-email",
+ tag_name="INPUT",
+ texts=["Email"],
+ input_type="email",
+ mouse=Mouse(xp=0.5, yp=0.5),
+ ),
+ timestamp_start=1000.0,
+ timestamp_end=1001.0,
+ url="https://example.com/login",
+ input_value="[email protected]",
+ )
+ )
+ assert "[email protected]" in captured["recording-action-block-prompt-input-text"]
+
+
+async def test_otp_autocomplete_ingest_redacts_and_stamps_totp(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:
+ raise RuntimeError("llm unavailable")
+
+ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)
+
+ session = RecordingInterpretationSession(
+ browser_session_id=PBS_ID,
+ organization_id=ORG_ID,
+ workflow_permanent_id=WP_ID,
+ on_update=lambda update: None,
+ debounce_seconds=0.01,
+ max_wait_seconds=0.05,
+ )
+ session.ingest_events(
+ make_streaming_console_input(
+ timestamp_ms=1000.0,
+ input_value="654321",
+ target_id="otp",
+ target_text="Code",
+ input_type="text",
+ autocomplete="one-time-code",
+ )
+ )
+ steps = await session.flush()
+
+ assert len(steps) == 1
+ assert steps[0].credential_kind == "totp"
+ assert "654321" not in (steps[0].navigation_goal or "")