Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Save credentials from the live draft (#8307)
Cindy Lieda820580d75
diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsxindex 90f61c2b5..7e42de134 100644--- a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.test.tsx@@ -4,12 +4,19 @@ import { cleanup, fireEvent, render, screen } from "@testing-library/react";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";-import { useRecordingStore } from "@/store/useRecordingStore";+import {+ useRecordingStore,+ type RecordingDraftStep,+} from "@/store/useRecordingStore";import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";import { RecordingPanel } from "./RecordingPanel";const mutateMock = vi.fn();+const modalState = vi.hoisted(() => ({+ overrideType: null as string | null,+ defaultTestUrl: null as string | null,+}));vi.mock("@/routes/browserSessions/hooks/useProcessRecordingMutation", () => ({useProcessRecordingMutation: () => ({@@ -19,14 +26,65 @@ vi.mock("@/routes/browserSessions/hooks/useProcessRecordingMutation", () => ({}),}));+vi.mock("@/routes/credentials/CredentialsModal", () => ({+ CredentialsModal: ({+ isOpen,+ onCredentialCreated,+ overrideType,+ defaultTestUrl,+ }: {+ isOpen?: boolean;+ onCredentialCreated?: (id: string, name?: string) => void;+ overrideType?: string;+ defaultTestUrl?: string;+ }) => {+ modalState.overrideType = overrideType ?? null;+ modalState.defaultTestUrl = defaultTestUrl ?? null;+ return isOpen ? (+ <button+ type="button"+ data-testid="mock-create-credential"+ onClick={() => onCredentialCreated?.("new-cred-1")}+ >+ create credential+ </button>+ ) : null;+ },+}));+const initialRecording = useRecordingStore.getState();const initialPanel = useWorkflowPanelStore.getState();const initialRecordedBlocks = useRecordedBlocksStore.getState();+function inputDraft(+ id: string,+ opts: {+ credential_kind?: RecordingDraftStep["credential_kind"];+ url?: string;+ title?: string;+ } = {},+): RecordingDraftStep {+ return {+ step_id: id,+ action_kind: "input_text",+ block_type: "action",+ label: id,+ title: opts.title ?? id,+ status: "ready",+ editable_fields: [],+ parameters: [],+ parameter_keys: [],+ url: opts.url ?? "https://example.com/login",+ credential_kind: opts.credential_kind,+ };+}+describe("RecordingPanel", () => {beforeEach(() => {HTMLElement.prototype.scrollIntoView = vi.fn();mutateMock.mockReset();+ modalState.overrideType = null;+ modalState.defaultTestUrl = null;useRecordingStore.setState(initialRecording, true);useWorkflowPanelStore.setState(initialPanel, true);useRecordedBlocksStore.setState(initialRecordedBlocks, true);@@ -113,4 +171,72 @@ describe("RecordingPanel", () => {vi.useRealTimers();}});++ it("shows add and dismiss on a password draft, not on an email draft", () => {+ useRecordingStore.setState({+ draftSteps: [+ inputDraft("email", { title: "Fill email" }),+ inputDraft("pw", {+ credential_kind: "password",+ title: "Fill password",+ }),+ ],+ });++ render(<RecordingPanel browserSessionId="pbs_123" />);++ expect(screen.getByText("Fill password")).toBeTruthy();+ expect(screen.getByText("Fill email")).toBeTruthy();+ expect(+ screen.getByRole("button", { name: /add to credentials/i }),+ ).toBeTruthy();+ expect(screen.getByRole("button", { name: /^dismiss$/i })).toBeTruthy();+ });++ it("hides the credential prompt after dismiss", () => {+ useRecordingStore.setState({+ draftSteps: [+ inputDraft("pw", {+ credential_kind: "password",+ title: "Fill password",+ }),+ ],+ });++ render(<RecordingPanel browserSessionId="pbs_123" />);+ fireEvent.click(screen.getByRole("button", { name: /^dismiss$/i }));++ expect(+ screen.queryByRole("button", { name: /add to credentials/i }),+ ).toBeNull();+ expect(screen.getByText("Fill password")).toBeTruthy();+ });++ it("opens the password credential modal and dismisses the prompt after create", () => {+ useRecordingStore.setState({+ draftSteps: [+ inputDraft("pw", {+ credential_kind: "password",+ title: "Fill password",+ url: "https://example.com/login",+ }),+ ],+ });++ render(<RecordingPanel browserSessionId="pbs_123" />);+ fireEvent.click(+ screen.getByRole("button", { name: /add to credentials/i }),+ );++ expect(modalState.overrideType).toBe("password");+ expect(modalState.defaultTestUrl).toBe("https://example.com/login");+ expect(screen.getByTestId("mock-create-credential")).toBeTruthy();++ fireEvent.click(screen.getByTestId("mock-create-credential"));++ expect(+ screen.queryByRole("button", { name: /add to credentials/i }),+ ).toBeNull();+ expect(screen.queryByTestId("mock-create-credential")).toBeNull();+ });});diff --git a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsxindex ea6970676..7f4295b52 100644--- a/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx+++ b/skyvern-frontend/src/routes/workflows/editor/recording/RecordingPanel.tsx@@ -22,6 +22,11 @@ import {} from "@/components/ui/dialog";import { useRecordingElapsedSeconds } from "@/hooks/useRecordingElapsedSeconds";import { useProcessRecordingMutation } from "@/routes/browserSessions/hooks/useProcessRecordingMutation";+import { CredentialsModal } from "@/routes/credentials/CredentialsModal";+import {+ CredentialModalTypes,+ type CredentialModalType,+} from "@/routes/credentials/useCredentialModalState";import { useRecordedBlocksStore } from "@/store/RecordedBlocksStore";import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";import {@@ -132,6 +137,14 @@ function StepScreenshot({ screenshot }: { screenshot: RecordingScreenshot }) {);}+function credentialModalTypeForKind(+ kind: NonNullable<RecordingDraftStep["credential_kind"]>,+): CredentialModalType {+ return kind === "credit_card"+ ? CredentialModalTypes.CREDIT_CARD+ : CredentialModalTypes.PASSWORD;+}+function DraftStepCard({step,index,@@ -139,6 +152,9 @@ function DraftStepCard({screenshot,onDelete,onRename,+ showCredentialPrompt,+ onAddCredentials,+ onDismissCredential,}: {step: RecordingDraftStep;index: number;@@ -146,6 +162,9 @@ function DraftStepCard({screenshot: RecordingScreenshot | null;onDelete: () => void;onRename: (value: string) => void;+ showCredentialPrompt: boolean;+ onAddCredentials: () => void;+ onDismissCredential: () => void;}) {const beginDraftEdit = useRecordingStore((state) => state.beginDraftEdit);const endDraftEdit = useRecordingStore((state) => state.endDraftEdit);@@ -267,6 +286,27 @@ function DraftStepCard({<StepScreenshot screenshot={screenshot} /></div>)}+ {showCredentialPrompt && (+ <div className="flex flex-wrap items-center gap-2 pl-8">+ <Button+ type="button"+ size="sm"+ className="h-7"+ onClick={onAddCredentials}+ >+ Add to credentials+ </Button>+ <Button+ type="button"+ variant="ghost"+ size="sm"+ className="h-7"+ onClick={onDismissCredential}+ >+ Dismiss+ </Button>+ </div>+ )}</div>);}@@ -335,6 +375,12 @@ function RecordingPanel({ browserSessionId }: Props) {// the mutation guard throw.const browserSessionMissing = !browserSessionId;const [confirmDiscardOpen, setConfirmDiscardOpen] = useState(false);+ const [credentialModal, setCredentialModal] = useState<{+ type: CredentialModalType;+ testUrl: string | null;+ url: string | null;+ stepId: string;+ } | null>(null);const feedEndRef = useRef<HTMLDivElement | null>(null);const committedRef = useRef(false);@@ -345,6 +391,7 @@ function RecordingPanel({ browserSessionId }: Props) {draftSteps,deletedStepIds,stepPatches,+ dismissedCredentialStepIds,screenshots,sessionRevision,optimisticSteps: rawOptimisticSteps,@@ -360,6 +407,7 @@ function RecordingPanel({ browserSessionId }: Props) {draftSteps: state.draftSteps,deletedStepIds: state.deletedStepIds,stepPatches: state.stepPatches,+ dismissedCredentialStepIds: state.dismissedCredentialStepIds,screenshots: state.screenshots,sessionRevision: state.sessionRevision,optimisticSteps: state.optimisticSteps,@@ -553,6 +601,26 @@ function RecordingPanel({ browserSessionId }: Props) {index={index}baselineMs={baselineMs}screenshot={findScreenshotForStep(step, screenshots)}+ showCredentialPrompt={+ Boolean(step.credential_kind) &&+ !dismissedCredentialStepIds.includes(step.step_id)+ }+ onAddCredentials={() => {+ if (!step.credential_kind) {+ return;+ }+ setCredentialModal({+ type: credentialModalTypeForKind(step.credential_kind),+ testUrl: step.url ?? null,+ url: step.url ?? null,+ stepId: step.step_id,+ });+ }}+ onDismissCredential={() =>+ useRecordingStore+ .getState()+ .dismissCredentialPrompt(step.step_id)+ }onDelete={() =>useRecordingStore.getState().deleteDraftStep(step.step_id)}@@ -671,6 +739,27 @@ function RecordingPanel({ browserSessionId }: Props) {</DialogContent></Dialog>)}+ {credentialModal ? (+ <CredentialsModal+ isOpen+ overrideType={credentialModal.type}+ defaultTestUrl={credentialModal.testUrl ?? undefined}+ onOpenChange={(open) => {+ if (!open) {+ setCredentialModal(null);+ }+ }}+ onCredentialCreated={() => {+ const store = useRecordingStore.getState();+ if (credentialModal.url) {+ store.dismissCredentialPromptsForUrl(credentialModal.url);+ } else {+ store.dismissCredentialPrompt(credentialModal.stepId);+ }+ setCredentialModal(null);+ }}+ />+ ) : null}</div>);}diff --git a/skyvern-frontend/src/store/useRecordingStore.test.ts b/skyvern-frontend/src/store/useRecordingStore.test.tsindex 855001f40..372f86a79 100644--- a/skyvern-frontend/src/store/useRecordingStore.test.ts+++ b/skyvern-frontend/src/store/useRecordingStore.test.ts@@ -373,3 +373,51 @@ describe("applyInterpretationUpdate: snapshot vs delta", () => {expect(store().draftSteps.map((s) => s.step_id)).toEqual(["s1-0"]);});});++describe("dismissCredentialPrompt", () => {+ it("hides the prompt for one step and clears on a new recording", () => {+ store().setIsRecording(true);+ store().applyInterpretationUpdate(+ update("s1", 1, [+ { ...draft("pw", "input_text"), credential_kind: "password" },+ { ...draft("otp", "input_text"), credential_kind: "totp" },+ ]),+ );++ store().dismissCredentialPrompt("pw");+ expect(store().dismissedCredentialStepIds).toEqual(["pw"]);++ store().dismissCredentialPrompt("pw");+ expect(store().dismissedCredentialStepIds).toEqual(["pw"]);++ store().reset();+ store().setIsRecording(true);+ expect(store().dismissedCredentialStepIds).toEqual([]);+ });++ it("dismisses every secret step on the same url", () => {+ store().setIsRecording(true);+ store().applyInterpretationUpdate(+ update("s1", 1, [+ {+ ...draft("pw", "input_text"),+ credential_kind: "password",+ url: "https://example.com/login",+ },+ {+ ...draft("otp", "input_text"),+ credential_kind: "totp",+ url: "https://example.com/login",+ },+ {+ ...draft("other", "input_text"),+ credential_kind: "password",+ url: "https://other.example/login",+ },+ ]),+ );++ store().dismissCredentialPromptsForUrl("https://example.com/login");+ expect(store().dismissedCredentialStepIds).toEqual(["pw", "otp"]);+ });+});diff --git a/skyvern-frontend/src/store/useRecordingStore.ts b/skyvern-frontend/src/store/useRecordingStore.tsindex 23e504e97..f0ca1e33d 100644--- a/skyvern-frontend/src/store/useRecordingStore.ts+++ b/skyvern-frontend/src/store/useRecordingStore.ts@@ -127,6 +127,7 @@ export interface RecordingDraftStep {parameter_keys: Array<string>;timestamp_start?: number | null;timestamp_end?: number | null;+ credential_kind?: "password" | "totp" | "credit_card" | null;}export interface RecordingInterpretationUpdate {@@ -285,6 +286,7 @@ interface RecordingStore {*/deletedStepIds: Array<string>;stepPatches: Record<string, RecordingDraftStepPatch>;+ dismissedCredentialStepIds: Array<string>;/*** Nested count of in-progress live draft title edits. While > 0, capture is* paused alongside manualCapturePaused.@@ -313,6 +315,8 @@ interface RecordingStore {addOptimisticStep: (step: OptimisticStep) => void;deleteDraftStep: (stepId: string) => void;patchDraftStep: (stepId: string, patch: RecordingDraftStepPatch) => void;+ dismissCredentialPrompt: (stepId: string) => void;+ dismissCredentialPromptsForUrl: (url: string | null | undefined) => void;beginDraftEdit: () => void;endDraftEdit: () => void;setManualCapturePaused: (paused: boolean) => void;@@ -511,6 +515,7 @@ function emptyRecordingState() {interpretationFinalized: false,deletedStepIds: [] as Array<string>,stepPatches: {} as Record<string, RecordingDraftStepPatch>,+ dismissedCredentialStepIds: [] as Array<string>,draftEditDepth: 0,manualCapturePaused: false,screenshots: [] as Array<RecordingScreenshot>,@@ -537,6 +542,7 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({interpretationFinalized: false,deletedStepIds: [],stepPatches: {},+ dismissedCredentialStepIds: [],draftEditDepth: 0,manualCapturePaused: false,screenshots: [],@@ -595,7 +601,13 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({...(clearOptimistic ? { optimisticSteps: [] } : {}),interpretationPending: update.pending,interpretationFinalized: update.finalized,- ...(sessionChanged ? { deletedStepIds: [], stepPatches: {} } : {}),+ ...(sessionChanged+ ? {+ deletedStepIds: [],+ stepPatches: {},+ dismissedCredentialStepIds: [],+ }+ : {}),});},@@ -635,6 +647,34 @@ export const useRecordingStore = create<RecordingStore>((set, get) => ({});},+ dismissCredentialPrompt: (stepId) => {+ const state = get();+ if (state.dismissedCredentialStepIds.includes(stepId)) {+ return;+ }+ set({+ dismissedCredentialStepIds: [...state.dismissedCredentialStepIds, stepId],+ });+ },++ dismissCredentialPromptsForUrl: (url) => {+ if (!url) {+ return;+ }+ const state = get();+ const dismissed = new Set(state.dismissedCredentialStepIds);+ for (const step of state.draftSteps) {+ if (!step.credential_kind || step.url !== url) {+ continue;+ }+ dismissed.add(step.step_id);+ }+ if (dismissed.size === state.dismissedCredentialStepIds.length) {+ return;+ }+ set({ dismissedCredentialStepIds: Array.from(dismissed) });+ },+beginDraftEdit: () => {set({ draftEditDepth: get().draftEditDepth + 1 });},diff --git a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.jsindex ca4a04059..692e3a81d 100644--- a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js+++ b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js@@ -120,6 +120,34 @@return isUniqueSelector(selector) ? selector : null;};+ const SECRET_INPUT_TYPES = new Set(["password"]);+ const SECRET_AUTOCOMPLETE_TOKENS = new Set([+ "current-password",+ "new-password",+ "one-time-code",+ "cc-number",+ "cc-csc",+ "cc-exp",+ "cc-exp-month",+ "cc-exp-year",+ ]);++ const isSecretField = (element) => {+ if (!element) {+ return false;+ }+ const inputType = String(element.type || "").toLowerCase();+ if (SECRET_INPUT_TYPES.has(inputType)) {+ return true;+ }+ const autocomplete = String(+ (element.getAttribute && element.getAttribute("autocomplete")) || "",+ ).toLowerCase();+ return autocomplete+ .split(/\s+/)+ .some((token) => SECRET_AUTOCOMPLETE_TOKENS.has(token));+ };+const IMPLICIT_INPUT_ROLES = {button: "button",submit: "button",@@ -346,7 +374,7 @@eventType,id: e.target?.id,className: e.target?.className,- value: e.target?.value,+ value: isSecretField(e.target) ? null : e.target?.value,text: getElementText(e.target),labels: getAssociatedLabels(e.target),skyId: e.target?.dataset?.skyId,@@ -368,6 +396,12 @@const classText = String(e.target.classList?.value ?? e.target.getAttribute("class") ?? "",);+ const secretField = isSecretField(e.target);+ const autocomplete = e.target?.getAttribute?.("autocomplete") || null;+ // Named keys ("Enter", "Tab") survive: the input-text state machine emits on Enter,+ // which is the only signal an Enter-submitted login produces before it navigates away.+ const redactKeystroke =+ secretField && typeof e.key === "string" && e.key.length === 1;const eventData = {url: window.location.href,@@ -379,7 +413,7 @@isHtml: e.target instanceof HTMLElement,isSvg: e.target instanceof SVGElement,className: classText,- value: e.target?.value,+ value: secretField ? null : e.target?.value,text: getElementText(e.target),labels: getAssociatedLabels(e.target),skyId: e.target?.dataset?.skyId,@@ -399,10 +433,13 @@e.target?.tagName === "BUTTON"? e.target?.type || null: null,+ autocomplete,},- inputValue: ["input", "focus", "blur"].includes(eventType)- ? e.target?.value- : undefined,+ inputValue: secretField+ ? null+ : ["input", "focus", "blur"].includes(eventType)+ ? e.target?.value+ : undefined,mousePosition: {xa: Number.isFinite(e.clientX) ? e.clientX : null,ya: Number.isFinite(e.clientY) ? e.clientY : null,@@ -415,8 +452,8 @@? e.clientY / window.innerHeight: null,},- key: e.key,- code: e.code,+ key: redactKeystroke ? null : e.key,+ code: redactKeystroke ? null : e.code,activeElement: {tagName: document.activeElement?.tagName,id: document.activeElement?.id,diff --git a/skyvern/services/browser_recording/interpretation.py b/skyvern/services/browser_recording/interpretation.pyindex d1bb98962..468bf6ae1 100644--- a/skyvern/services/browser_recording/interpretation.py+++ b/skyvern/services/browser_recording/interpretation.py@@ -16,6 +16,7 @@ from skyvern.client.types.workflow_definition_yaml_blocks_item import (from skyvern.config import settingsfrom skyvern.forge.sdk.routes.streaming.channels import exfiltration as streaming_exfiltrationfrom skyvern.forge.sdk.routes.streaming.channels.exfiltration import ExfiltratedEventSource+from skyvern.services.browser_recording.redact import credential_kind_for_target, redact_console_eventfrom skyvern.services.browser_recording.service import (Processor,deterministic_input_text_parameter_key,@@ -77,7 +78,7 @@ def streaming_events_to_recording_events(if source == "cdp":reified_events.append(ExfiltratedCdpEvent(**payload))elif source == "console":- reified_events.append(ExfiltratedConsoleEvent(**payload))+ reified_events.append(redact_console_event(ExfiltratedConsoleEvent(**payload)))else:LOG.debug("Skipping recording interpretation event with unsupported source", source=source)except Exception:@@ -110,6 +111,12 @@ def _extra_field(block: OutputBlock, field_name: str, fallback: t.Any) -> t.Any:return fallback+def _credential_kind_for_action(action: Action) -> t.Literal["password", "totp", "credit_card"] | None:+ if not isinstance(action, ActionInputText):+ return None+ return credential_kind_for_target(action.target.input_type, action.target.autocomplete)++def _draft_step_from_block(*,browser_session_id: str,@@ -138,6 +145,7 @@ def _draft_step_from_block(parameter_keys=block.parameter_keys or [],timestamp_start=action.timestamp_start,timestamp_end=action.timestamp_end,+ credential_kind=_credential_kind_for_action(action),)if isinstance(block, WorkflowDefinitionYamlBlocksItem_GotoUrl):@@ -223,6 +231,7 @@ def _placeholder_step_from_action(label=normalize_recording_block_label(f"{action.kind.value}_{text}", fallback=action.kind.value),title=title,navigation_goal=navigation_goal,+ url=action.url,status=RecordingDraftStepStatus.INTERPRETING,editable_fields=[RecordingDraftStepEditableField.LABEL,@@ -233,6 +242,7 @@ def _placeholder_step_from_action(parameter_keys=parameter_keys,timestamp_start=action.timestamp_start,timestamp_end=action.timestamp_end,+ credential_kind=_credential_kind_for_action(action),)@@ -508,6 +518,10 @@ class RecordingInterpretationSession:step.navigation_goal = enriched.navigation_goalstep.parameters = enriched.parametersstep.parameter_keys = enriched.parameter_keys+ if enriched.credential_kind:+ step.credential_kind = enriched.credential_kind+ if enriched.url:+ step.url = enriched.urlstep.status = RecordingDraftStepStatus.READYself._emit(changed_steps=[step])diff --git a/skyvern/services/browser_recording/redact.py b/skyvern/services/browser_recording/redact.pynew file mode 100644index 000000000..33cb07956--- /dev/null+++ b/skyvern/services/browser_recording/redact.py@@ -0,0 +1,87 @@+"""Secret-field detection for record-browser capture and live drafts.++Keep the token lists identical to `isSecretField` in+`skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js`.+"""++from __future__ import annotations++import typing as t++from skyvern.services.browser_recording.types import ExfiltratedConsoleEvent, ExfiltratedEvent++CredentialKind = t.Literal["password", "totp", "credit_card"]++SECRET_INPUT_TYPES = frozenset({"password"})+SECRET_AUTOCOMPLETE_TOKENS = frozenset(+ {+ "current-password",+ "new-password",+ "one-time-code",+ "cc-number",+ "cc-csc",+ "cc-exp",+ "cc-exp-month",+ "cc-exp-year",+ }+)+PASSWORD_AUTOCOMPLETE_TOKENS = frozenset({"current-password", "new-password"})+CREDIT_CARD_AUTOCOMPLETE_TOKENS = frozenset(+ {+ "cc-number",+ "cc-csc",+ "cc-exp",+ "cc-exp-month",+ "cc-exp-year",+ }+)+++def _autocomplete_tokens(autocomplete: str | None) -> list[str]:+ if not autocomplete:+ return []+ return autocomplete.lower().split()+++def is_secret_field(input_type: str | None, autocomplete: str | None) -> bool:+ if (input_type or "").lower() in SECRET_INPUT_TYPES:+ return True+ return any(token in SECRET_AUTOCOMPLETE_TOKENS for token in _autocomplete_tokens(autocomplete))+++def credential_kind_for_target(input_type: str | None, autocomplete: str | None) -> CredentialKind | None:+ tokens = _autocomplete_tokens(autocomplete)+ if any(token in CREDIT_CARD_AUTOCOMPLETE_TOKENS for token in tokens):+ return "credit_card"+ if "one-time-code" in tokens:+ return "totp"+ if (input_type or "").lower() == "password" or any(token in PASSWORD_AUTOCOMPLETE_TOKENS for token in tokens):+ return "password"+ return None+++def is_character_key(key: str | None) -> bool:+ """Whether a KeyboardEvent.key carries typed content rather than naming a key.++ Printable keys are a single character ("a", "1"); named keys are longer ("Enter", "Tab").+ Only the former can reconstruct a secret, so only the former is dropped.+ """+ return key is not None and len(key) == 1+++def redact_console_event(event: ExfiltratedEvent) -> ExfiltratedEvent:+ if not isinstance(event, ExfiltratedConsoleEvent):+ return event++ target = event.params.target+ if not is_secret_field(target.inputType, target.autocomplete):+ return event++ target.value = None+ event.params.inputValue = None+ # Named keys survive: StateMachineInputText emits on Enter, which is the only signal a+ # login submitted with Enter (never blurring before navigation) ever produces.+ if is_character_key(event.params.key):+ event.params.key = None+ event.params.code = None+ return eventdiff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.pyindex 4f32b0756..fe3416327 100644--- a/skyvern/services/browser_recording/service.py+++ b/skyvern/services/browser_recording/service.py@@ -24,6 +24,7 @@ from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandlerfrom skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactoryfrom skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistryfrom skyvern.services.browser_recording.code_first import actions_to_code_first_blocks+from skyvern.services.browser_recording.redact import is_secret_field, redact_console_eventfrom skyvern.services.browser_recording.types import (Action,ActionBlockable,@@ -330,7 +331,7 @@ class Processor:else:LOG.error(f"{self.class_name} Unknown event source: {event.get('source')}", **self.identity)continue- reified_events.append(reified_event)+ reified_events.append(redact_console_event(reified_event))return reified_events@@ -588,6 +589,12 @@ class Processor:if action.kind == ActionKind.INPUT_TEXT:prompt_name = "recording-action-block-prompt-input-text"+ if (+ isinstance(action, ActionInputText)+ and is_secret_field(action.target.input_type, action.target.autocomplete)+ and action.input_value+ ):+ action = action.model_copy(update={"input_value": ""})else:prompt_name = "recording-action-block-prompt"diff --git a/skyvern/services/browser_recording/state_machines/input_text.py b/skyvern/services/browser_recording/state_machines/input_text.pyindex c1b799a16..b0b1a2780 100644--- a/skyvern/services/browser_recording/state_machines/input_text.py+++ b/skyvern/services/browser_recording/state_machines/input_text.py@@ -2,6 +2,7 @@ import typing as timport structlog+from skyvern.services.browser_recording.redact import is_secret_fieldfrom skyvern.services.browser_recording.types import (Action,ActionInputText,@@ -101,11 +102,12 @@ class StateMachineInputText(StateMachine):xp = (self.mouse.xp or -1) if self.mouse else Noneyp = (self.mouse.yp or -1) if self.mouse else None- LOG.debug("~ emitting input text action", exfiltrated_event=event)+ LOG.debug("~ emitting input text action")+ secret = is_secret_field(self.target.inputType, self.target.autocomplete)input_value = event.params.target.value- if input_value is None:+ if input_value is None and not secret:LOG.debug("~ cannot emit, missing input value; resetting")self.reset()@@ -122,6 +124,7 @@ class StateMachineInputText(StateMachine):role=self.target.role,accessible_name=self.target.accessibleName,input_type=self.target.inputType,+ autocomplete=self.target.autocomplete,)action = ActionInputText(@@ -130,7 +133,7 @@ class StateMachineInputText(StateMachine):timestamp_start=self.timestamp_start,timestamp_end=event.params.timestamp,url=event.params.url,- input_value=str(input_value),+ input_value="" if secret else str(input_value),)self.reset()diff --git a/skyvern/services/browser_recording/types.py b/skyvern/services/browser_recording/types.pyindex adf26958d..71de88ccd 100644--- a/skyvern/services/browser_recording/types.py+++ b/skyvern/services/browser_recording/types.py@@ -92,6 +92,7 @@ class ActionTarget(BaseModel):role: str | None = Noneaccessible_name: str | None = Noneinput_type: str | None = None+ autocomplete: str | None = Noneclass Mouse(BaseModel):@@ -133,6 +134,7 @@ class RecordingDraftStep(BaseModel):# locally-captured artifacts (e.g. stream screenshots) with each step.timestamp_start: float | None = Nonetimestamp_end: float | None = None+ credential_kind: Literal["password", "totp", "credit_card"] | None = Noneclass RecordingInterpretationUpdate(BaseModel):@@ -195,6 +197,7 @@ class EventTarget(BaseModel):role: str | None = NoneaccessibleName: str | None = NoneinputType: str | None = None+ autocomplete: str | None = Noneclass MousePosition(BaseModel):diff --git a/tests/unit/services/test_browser_recording.py b/tests/unit/services/test_browser_recording.pyindex 2515a3b71..7506f4db2 100644--- a/tests/unit/services/test_browser_recording.py+++ b/tests/unit/services/test_browser_recording.py@@ -268,14 +268,20 @@ def make_streaming_console_input(input_value: str,target_id: str = "email",target_text: str = "Email",+ input_type: str | None = None,+ autocomplete: str | None = None,) -> list[StreamingExfiltratedEvent]:- target = {+ target: dict[str, t.Any] = {"id": target_id,"skyId": "sky-email","tagName": "INPUT","text": [target_text],"value": input_value,}+ if input_type is not None:+ target["inputType"] = input_type+ if autocomplete is not None:+ target["autocomplete"] = autocompletecommon = {"target": target,"timestamp": timestamp_ms,@@ -884,3 +890,299 @@ def test_decompress_returns_bytes_for_valid_payload() -> None:payload = base64.b64encode(gzip.compress(raw)).decode("ascii")assert processor.decompress(payload) == raw+++def make_keydown_event(target: dict[str, t.Any], timestamp: float, key: str = "a") -> ExfiltratedConsoleEvent:+ return make_console_event(+ params={"type": "keydown", "key": key, "target": target, "timestamp": timestamp},+ timestamp=timestamp,+ )+++def make_blur_event(target: dict[str, t.Any], timestamp: float) -> ExfiltratedConsoleEvent:+ return make_console_event(+ params={"type": "blur", "target": target, "timestamp": timestamp},+ timestamp=timestamp,+ )+++def test_is_secret_field_and_credential_kind() -> None:+ from skyvern.services.browser_recording.redact import credential_kind_for_target, is_secret_field++ assert is_secret_field("password", None) is True+ assert is_secret_field("text", "one-time-code") is True+ assert is_secret_field("text", "cc-number") is True+ assert is_secret_field("text", "username current-password") is True+ assert is_secret_field("email", None) is False+ assert is_secret_field("text", "off") is False+ assert is_secret_field("text", "username") is False++ assert credential_kind_for_target("password", None) == "password"+ assert credential_kind_for_target("text", "current-password") == "password"+ assert credential_kind_for_target("text", "one-time-code") == "totp"+ assert credential_kind_for_target("text", "cc-number") == "credit_card"+ assert credential_kind_for_target("email", None) is None+ assert credential_kind_for_target("text", "off") is None+++def test_reify_strips_secret_values_from_console_events() -> None:+ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ events = processor.reify(+ [+ {+ "kind": "exfiltrated-event",+ "event_name": "user_interaction",+ "source": "console",+ "timestamp": 1.0,+ "params": {+ "type": "keydown",+ "key": "h",+ "code": "KeyH",+ "inputValue": "hunter2",+ "url": "https://example.com/login",+ "timestamp": 1000.0,+ "target": {+ "tagName": "INPUT",+ "id": "password",+ "skyId": "sky-pw",+ "inputType": "password",+ "value": "hunter2",+ "text": ["Password"],+ },+ "activeElement": {"tagName": "INPUT"},+ "window": {"height": 800, "width": 1200, "scrollX": 0, "scrollY": 0},+ "mousePosition": {"xp": 0.5, "yp": 0.5},+ },+ }+ ]+ )++ assert len(events) == 1+ event = events[0]+ assert isinstance(event, ExfiltratedConsoleEvent)+ assert event.params.target.value is None+ assert event.params.inputValue is None+ assert event.params.key is None+ assert event.params.code is None+ assert event.params.target.inputType == "password"+++def test_password_fill_still_emits_empty_input_text() -> None:+ target = {+ "id": "password",+ "skyId": "sky-pw",+ "tagName": "INPUT",+ "text": ["Password"],+ "inputType": "password",+ "value": None,+ }+ events = [+ make_focus_event(target=target, timestamp=1000.0),+ make_keydown_event(target=target, timestamp=1001.0),+ make_blur_event(target=target, timestamp=1002.0),+ ]++ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ actions = processor.events_to_actions(events)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, ActionInputText)+ assert action.input_value == ""+ assert action.target.input_type == "password"+++def test_password_submitted_with_enter_still_emits_input_text() -> None:+ """Enter-to-submit never blurs before navigating, so the Enter keydown is the only emit signal.++ Redacting it along with the character keystrokes silently drops the whole password step.+ """+ target = {+ "id": "password",+ "skyId": "sky-pw",+ "tagName": "INPUT",+ "text": ["Password"],+ "inputType": "password",+ "value": None,+ }+ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ # Enter through reify(), not events_to_actions(): ingest redaction runs there, so a test+ # below it would pass even with the Enter keydown stripped.+ events = processor.reify(+ [+ make_focus_event(target=target, timestamp=1000.0).model_dump(),+ make_keydown_event(target=target, timestamp=1001.0, key="s").model_dump(),+ make_keydown_event(target=target, timestamp=1002.0, key="Enter").model_dump(),+ ]+ )+ actions = processor.events_to_actions(events)++ assert len(actions) == 1+ action = actions[0]+ assert isinstance(action, ActionInputText)+ assert action.input_value == ""+ assert action.target.input_type == "password"+++def test_secret_character_keystrokes_still_redacted() -> None:+ from skyvern.services.browser_recording.redact import redact_console_event++ target = {"id": "password", "tagName": "INPUT", "inputType": "password", "value": "hunter2"}++ typed = redact_console_event(make_keydown_event(target=target, timestamp=1.0, key="s"))+ assert typed.params.key is None+ assert typed.params.code is None+ assert typed.params.target.value is None++ submit = redact_console_event(make_keydown_event(target=target, timestamp=2.0, key="Enter"))+ assert submit.params.key == "Enter"+ assert submit.params.target.value is None+++def test_non_secret_empty_value_still_skipped() -> None:+ target = {+ "id": "email",+ "skyId": "sky-email",+ "tagName": "INPUT",+ "text": ["Email"],+ "inputType": "email",+ "value": None,+ }+ events = [+ make_focus_event(target=target, timestamp=1000.0),+ make_keydown_event(target=target, timestamp=1001.0),+ make_blur_event(target=target, timestamp=1002.0),+ ]++ processor = Processor(PBS_ID, ORG_ID, WP_ID)+ assert processor.events_to_actions(events) == []+++async def test_password_ingest_emits_redacted_draft_with_credential_kind(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:+ raise RuntimeError("llm unavailable")++ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)++ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda update: None,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ )++ session.ingest_events(+ make_streaming_console_input(+ timestamp_ms=1000.0,+ input_value="hunter2",+ target_id="password",+ target_text="Password",+ input_type="password",+ )+ )+ steps = await session.flush()++ assert len(steps) == 1+ assert steps[0].credential_kind == "password"+ assert "hunter2" not in (steps[0].title or "")+ assert "hunter2" not in (steps[0].navigation_goal or "")+ stored = session.events+ for event in stored:+ if isinstance(event, ExfiltratedConsoleEvent):+ assert event.params.target.value != "hunter2"+ assert event.params.inputValue != "hunter2"+++async def test_create_action_block_prompt_omits_secret_keeps_email(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ import skyvern.services.browser_recording.service as svc++ captured: dict[str, str] = {}++ async def fake_llm(*, prompt: str, prompt_name: str, organization_id: str) -> dict[str, t.Any]:+ captured[prompt_name] = prompt+ return {"block_label": "fill", "title": "Fill", "prompt": "Fill the field."}++ monkeypatch.setattr(svc, "_recording_enrichment_llm_handler", lambda: fake_llm)++ processor = Processor(PBS_ID, ORG_ID, WP_ID)++ await processor.create_action_block(+ ActionInputText(+ kind=ActionKind.INPUT_TEXT,+ target=ActionTarget(+ id="password",+ sky_id="sky-pw",+ tag_name="INPUT",+ texts=["Password"],+ input_type="password",+ mouse=Mouse(xp=0.5, yp=0.5),+ ),+ timestamp_start=1000.0,+ timestamp_end=1001.0,+ url="https://example.com/login",+ input_value="hunter2",+ )+ )+ assert "hunter2" not in captured["recording-action-block-prompt-input-text"]++ captured.clear()+ await processor.create_action_block(+ ActionInputText(+ kind=ActionKind.INPUT_TEXT,+ target=ActionTarget(+ id="email",+ sky_id="sky-email",+ tag_name="INPUT",+ texts=["Email"],+ input_type="email",+ mouse=Mouse(xp=0.5, yp=0.5),+ ),+ timestamp_start=1000.0,+ timestamp_end=1001.0,+ url="https://example.com/login",+ input_value="[email protected]",+ )+ )+ assert "[email protected]" in captured["recording-action-block-prompt-input-text"]+++async def test_otp_autocomplete_ingest_redacts_and_stamps_totp(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def failing_llm(*args: t.Any, **kwargs: t.Any) -> dict[str, t.Any]:+ raise RuntimeError("llm unavailable")++ monkeypatch.setattr(app, "LLM_API_HANDLER", failing_llm)++ session = RecordingInterpretationSession(+ browser_session_id=PBS_ID,+ organization_id=ORG_ID,+ workflow_permanent_id=WP_ID,+ on_update=lambda update: None,+ debounce_seconds=0.01,+ max_wait_seconds=0.05,+ )+ session.ingest_events(+ make_streaming_console_input(+ timestamp_ms=1000.0,+ input_value="654321",+ target_id="otp",+ target_text="Code",+ input_type="text",+ autocomplete="one-time-code",+ )+ )+ steps = await session.flush()++ assert len(steps) == 1+ assert steps[0].credential_kind == "totp"+ assert "654321" not in (steps[0].navigation_goal or "")