Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(browser): redact encoded declared secrets consistently
morlutof09bb8a99773
diff --git a/src/browser/BrowserScenario.test.ts b/src/browser/BrowserScenario.test.tsindex 2262f410a..ee736ee5e 100644--- a/src/browser/BrowserScenario.test.ts+++ b/src/browser/BrowserScenario.test.ts@@ -80,6 +80,49 @@ const baseScenario = () => ({},});+describe("browser scenario secret redaction", () => {+ it("matches percent-escape hex case without case-folding raw secrets", () => {+ const secrets = BrowserScenarioSecrets.resolve(+ browserScenarioSchema.parse(baseScenario()),+ {+ REA_TEST_SESSION: "a/b?",+ REA_TEST_PASSWORD: "password",+ REA_TEST_COOKIE: "cookie",+ },+ );+ if (secrets === undefined) throw new Error("Expected resolved secrets");++ expect(secrets.redact("logged a%2fb%3f, a%2Fb%3F and a%2fb%3F")).toBe(+ "logged [REDACTED:session], [REDACTED:session] and [REDACTED:session]",+ );+ expect(secrets.redact("raw A/B? remains case-sensitive")).toBe(+ "raw A/B? remains case-sensitive",+ );+ expect(+ secrets.redactBytes(+ Buffer.concat([+ Buffer.from([0xff, 0]),+ Buffer.from("a%2fb%3F"),+ Buffer.from([0x80]),+ ]),+ ),+ ).toEqual(+ Buffer.concat([+ Buffer.from([0xff, 0]),+ Buffer.from("[REDACTED:session]"),+ Buffer.from([0x80]),+ ]),+ );+ expect(+ secrets.sanitizeUrl("https://app.example.test/?token=a%2fb%3F").url,+ ).toBe("https://app.example.test/?token=[REDACTED:session]");+ expect(secrets.fingerprint("stored a%2fb%3F")).toEqual({+ value_state: "redacted-secret",+ value_sha256: null,+ });+ });+});+describe("browserScenarioSchema", () => {it("accepts and normalizes a declared scenario", () => {const parsed = browserScenarioSchema.parse(baseScenario());diff --git a/src/browser/BrowserScenarioSecrets.ts b/src/browser/BrowserScenarioSecrets.tsindex 05fb99047..aa583c016 100644--- a/src/browser/BrowserScenarioSecrets.ts+++ b/src/browser/BrowserScenarioSecrets.ts@@ -15,84 +15,229 @@ const REDACTION_PREFIX = "[REDACTED:";const encodedSecretValues = (secret: string): ReadonlySet<string> =>new Set([- secret,encodeURIComponent(secret),new URLSearchParams([["value", secret]]).toString().slice("value=".length),]);-type SecretSpellings = "raw" | "url" | "bytes";+type SecretSpellings = "url" | "text";+interface SecretPattern {+ readonly text: string;+ readonly searchText: string;+ readonly percentInsensitive: boolean;+}+interface SecretReplacement extends SecretPattern {+ readonly id: string;+}+interface SecretNeedle {+ readonly id: string;+ readonly needle: Buffer;+ readonly percentInsensitive: boolean;+ readonly marker: Buffer;+}-const secretSpellings = (+const normalizePercentEscapes = (value: string): string => {+ if (!value.includes("%")) return value;+ return value.replace(/%[0-9a-f]{2}/giu, (escape) => escape.toLowerCase());+};++const secretPatterns = (secret: string,spellings: SecretSpellings,-): readonly string[] => {+): readonly SecretPattern[] => {if (secret === "") return [];- if (spellings === "raw") return [secret];- const values = [...encodedSecretValues(secret)];- if (spellings === "bytes") values.push(JSON.stringify(secret).slice(1, -1));- return [...new Set(values)].filter((value) => value !== "");+ const patterns: SecretPattern[] = [+ { text: secret, searchText: secret, percentInsensitive: false },+ ...[...encodedSecretValues(secret)].map((text) => ({+ text,+ searchText: normalizePercentEscapes(text),+ percentInsensitive: true,+ })),+ ];+ if (spellings === "text")+ patterns.push({+ text: JSON.stringify(secret).slice(1, -1),+ searchText: JSON.stringify(secret).slice(1, -1),+ percentInsensitive: false,+ });+ const seen = new Set<string>();+ return patterns.filter(({ text, percentInsensitive }) => {+ if (text === "") return false;+ const key = `${percentInsensitive ? "percent" : "literal"}:${text}`;+ if (seen.has(key)) return false;+ seen.add(key);+ return true;+ });};-const declaredLiterals = (+const declaredPatterns = (values: ReadonlyMap<string, string>,spellings: SecretSpellings,-): readonly string[] => [- ...new Set(- [...values.values()].flatMap((secret) =>- secretSpellings(secret, spellings),- ),- ),-];--/** Pick a marker that does not itself contain a declared secret spelling. */-const redactionMarker = (id: string, literals: readonly string[]): string =>- [`${REDACTION_PREFIX}${id}]`, "[REDACTED]", "…", ""].find((candidate) =>- literals.every((literal) => !candidate.includes(literal)),- ) ?? "";+): readonly SecretPattern[] => {+ const seen = new Set<string>();+ return [...values.values()]+ .flatMap((secret) => secretPatterns(secret, spellings))+ .filter(({ text, percentInsensitive }) => {+ const key = `${percentInsensitive ? "percent" : "literal"}:${text}`;+ if (seen.has(key)) return false;+ seen.add(key);+ return true;+ });+};-const replaceSecretLiterals = (- value: string,+const secretReplacements = (values: ReadonlyMap<string, string>,spellings: SecretSpellings,- literals: readonly string[],-): string => {- const replacements = [...values]+): readonly SecretReplacement[] =>+ [...values].flatMap(([id, secret]) =>- secretSpellings(secret, spellings).map((text) => ({ id, text })),+ secretPatterns(secret, spellings).map((pattern) => ({ id, ...pattern })),).sort((left, right) =>right.text.length - left.text.length ||(left.id < right.id ? -1 : left.id > right.id ? 1 : 0),);- let output = value;- for (const replacement of replacements)- output = output.replaceAll(- replacement.text,- redactionMarker(replacement.id, literals),- );- return output;++const normalizePercentEscapesBytes = (value: Buffer): Buffer => {+ if (value.indexOf(37) < 0) return value;+ const normalized = Buffer.from(value);+ for (let index = 0; index + 2 < normalized.length; index += 1)+ if (+ normalized[index] === 37 &&+ isHexByte(normalized[index + 1]) &&+ isHexByte(normalized[index + 2])+ ) {+ normalized[index + 1] = lowercaseHexByte(normalized[index + 1]!);+ normalized[index + 2] = lowercaseHexByte(normalized[index + 2]!);+ index += 2;+ }+ return normalized;};-const secretNeedles = (- values: ReadonlyMap<string, string>,- literals: readonly string[],-): Array<{- readonly id: string;- readonly needle: Buffer;- readonly marker: Buffer;-}> =>- [...values].flatMap(([id, secret]) =>- secretSpellings(secret, "bytes").map((text) => ({- id,- needle: Buffer.from(text),- marker: Buffer.from(redactionMarker(id, literals)),- })),+const isHexByte = (value: number | undefined): boolean =>+ value !== undefined &&+ ((value >= 48 && value <= 57) ||+ (value >= 65 && value <= 70) ||+ (value >= 97 && value <= 102));++const lowercaseHexByte = (value: number): number =>+ value >= 65 && value <= 70 ? value + 32 : value;++const patternMatches = (+ value: string,+ normalizedValue: string,+ pattern: SecretPattern,+): boolean =>+ (pattern.percentInsensitive ? normalizedValue : value).includes(+ pattern.searchText,);+const containsSecretPattern = (+ value: string,+ normalizedValue: string,+ patterns: readonly SecretPattern[],+): boolean => {+ if (patterns.length === 0) return false;+ return patterns.some((pattern) =>+ patternMatches(value, normalizedValue, pattern),+ );+};++const containsSecretBytePattern = (+ value: Buffer,+ normalizedValue: Buffer,+ needles: readonly SecretNeedle[],+): boolean =>+ needles.some((needle) =>+ (needle.percentInsensitive ? normalizedValue : value).includes(+ needle.needle,+ ),+ );++/** Pick a marker that does not itself contain a declared secret spelling. */+const redactionMarker = (+ id: string,+ patterns: readonly SecretPattern[],+): string => {+ const candidates = [`${REDACTION_PREFIX}${id}]`, "[REDACTED]", "…", ""];+ return (+ candidates.find((candidate) =>+ patterns.every(+ (pattern) =>+ !patternMatches(+ candidate,+ normalizePercentEscapes(candidate),+ pattern,+ ),+ ),+ ) ?? ""+ );+};++const replaceSecretLiterals = (+ value: string,+ replacements: readonly SecretReplacement[],+ patterns: readonly SecretPattern[],+): string => {+ const normalizedValue = replacements.some(+ ({ percentInsensitive }) => percentInsensitive,+ )+ ? normalizePercentEscapes(value)+ : value;+ const parts: string[] = [];+ let offset = 0;+ while (offset < value.length) {+ let next:+ | ((typeof replacements)[number] & { readonly index: number })+ | undefined;+ for (const replacement of replacements) {+ const search = replacement.searchText;+ const view = replacement.percentInsensitive ? normalizedValue : value;+ const index = view.indexOf(search, offset);+ if (+ index >= 0 &&+ (next === undefined ||+ index < next.index ||+ (index === next.index && replacement.text.length > next.text.length))+ )+ next = { ...replacement, index };+ }+ if (next === undefined) break;+ const marker = redactionMarker(next.id, patterns);+ parts.push(value.slice(offset, next.index), marker);+ offset = next.index + next.text.length;+ }+ if (parts.length === 0) return value;+ parts.push(value.slice(offset));+ return parts.join("");+};++const secretNeedles = (+ replacements: readonly SecretReplacement[],+ patterns: readonly SecretPattern[],+): SecretNeedle[] =>+ replacements.map((pattern) => ({+ id: pattern.id,+ needle: Buffer.from(pattern.searchText),+ percentInsensitive: pattern.percentInsensitive,+ marker: Buffer.from(redactionMarker(pattern.id, patterns)),+ }));+/** Resolved secret values kept only for one in-memory scenario session. */export class BrowserScenarioSecrets {- private constructor(private readonly values: ReadonlyMap<string, string>) {}+ private readonly urlPatterns: readonly SecretPattern[];+ private readonly textPatterns: readonly SecretPattern[];+ private readonly urlReplacements: readonly SecretReplacement[];+ private readonly textReplacements: readonly SecretReplacement[];+ private readonly textNeedles: readonly SecretNeedle[];++ private constructor(private readonly values: ReadonlyMap<string, string>) {+ this.urlPatterns = declaredPatterns(values, "url");+ this.textPatterns = declaredPatterns(values, "text");+ this.urlReplacements = secretReplacements(values, "url");+ this.textReplacements = secretReplacements(values, "text");+ this.textNeedles = secretNeedles(this.textReplacements, this.textPatterns);+ }static resolve(scenario: BrowserScenario,@@ -127,18 +272,40 @@ export class BrowserScenarioSecrets {}redact(value: string): string {- const literals = declaredLiterals(this.values, "raw");- const output = replaceSecretLiterals(value, this.values, "raw", literals);- return literals.some((literal) => output.includes(literal)) ? "" : output;+ const patterns = this.textPatterns;+ if (patterns.length === 0) return value;+ const output = replaceSecretLiterals(+ value,+ this.textReplacements,+ patterns,+ );+ return containsSecretPattern(+ output,+ normalizePercentEscapes(output),+ patterns,+ )+ ? ""+ : output;}- /** Replace declared UTF-8, URI/form, and JSON-escaped secrets without decoding binary data. */+ /** Replace declared UTF-8, URI/form, and JSON-escaped secret spellings. */redactBytes(value: Buffer): Buffer {- const literals = declaredLiterals(this.values, "bytes");- const candidates = secretNeedles(this.values, literals)+ const patterns = this.textPatterns;+ if (patterns.length === 0) return value;+ const candidates = this.textNeedles;+ const hasPercentPatterns = candidates.some(+ ({ percentInsensitive }) => percentInsensitive,+ );+ const normalizedValue =+ hasPercentPatterns && value.indexOf(37) >= 0+ ? normalizePercentEscapesBytes(value)+ : value;+ const sortedCandidates = candidates.map((candidate) => ({...candidate,- index: value.indexOf(candidate.needle),+ index: (candidate.percentInsensitive ? normalizedValue : value).indexOf(+ candidate.needle,+ ),})).sort((left, right) =>@@ -148,13 +315,18 @@ export class BrowserScenarioSecrets {const parts: Buffer[] = [];let offset = 0;while (true) {- let next: (typeof candidates)[number] | undefined;- for (const candidate of candidates) {+ let next: (typeof sortedCandidates)[number] | undefined;+ for (const candidate of sortedCandidates) {if (candidate.index >= 0 && candidate.index < offset)- candidate.index = value.indexOf(candidate.needle, offset);+ candidate.index = (+ candidate.percentInsensitive ? normalizedValue : value+ ).indexOf(candidate.needle, offset);if (candidate.index >= 0 &&- (next === undefined || candidate.index < next.index)+ (next === undefined ||+ candidate.index < next.index ||+ (candidate.index === next.index &&+ candidate.needle.length > next.needle.length)))next = candidate;}@@ -165,16 +337,23 @@ export class BrowserScenarioSecrets {if (parts.length === 0) return value;parts.push(value.subarray(offset));const redacted = Buffer.concat(parts);- return literals.some((literal) => redacted.includes(Buffer.from(literal)))+ const normalizedRedacted =+ hasPercentPatterns && redacted.indexOf(37) >= 0+ ? normalizePercentEscapesBytes(redacted)+ : redacted;+ return containsSecretBytePattern(redacted, normalizedRedacted, candidates)? Buffer.alloc(0): redacted;}/** Remove only declared secret values from one URL and report that action. */sanitizeUrl(value: string): SanitizedBrowserUrl {- const literals = declaredLiterals(this.values, "url");- const output = replaceSecretLiterals(value, this.values, "url", literals);- if (literals.some((literal) => output.includes(literal)))+ const patterns = this.urlPatterns;+ if (patterns.length === 0) return sanitizeBrowserUrl(value);+ const output = replaceSecretLiterals(value, this.urlReplacements, patterns);+ if (+ containsSecretPattern(output, normalizePercentEscapes(output), patterns)+ )return {url: "",origin: null,@@ -189,14 +368,15 @@ export class BrowserScenarioSecrets {}fingerprint(value: string): BrowserStorageValueFingerprint {- for (const secret of this.values.values())- if (secret !== "" && value.includes(secret))- return { value_state: "redacted-secret", value_sha256: null };+ const patterns = this.textPatterns;+ if (+ patterns.length > 0 &&+ containsSecretPattern(value, normalizePercentEscapes(value), patterns)+ )+ return { value_state: "redacted-secret", value_sha256: null };return {value_state: "hashed",- value_sha256: createHash("sha256")- .update(this.redact(value))- .digest("hex"),+ value_sha256: createHash("sha256").update(value).digest("hex"),};}}diff --git a/src/browser/PlaywrightScenarioEvents.test.ts b/src/browser/PlaywrightScenarioEvents.test.tsindex 4c96e8d3d..4e7841365 100644--- a/src/browser/PlaywrightScenarioEvents.test.ts+++ b/src/browser/PlaywrightScenarioEvents.test.ts@@ -5,6 +5,77 @@ import { browserScenarioSchema } from "../domain/browserScenario.js";import { BrowserScenarioSecrets } from "./BrowserScenarioSecrets.js";import { PlaywrightScenarioEvents } from "./PlaywrightScenarioEvents.js";+describe("declared secrets in scenario events", () => {+ it("redacts encoded declared secrets from text evidence and fingerprints", () => {+ const listeners = new Map<string, (value: unknown) => void>();+ const socketListeners = new Map<string, (value: unknown) => void>();+ const page = {+ on: (name: string, listener: (value: unknown) => void) => {+ listeners.set(name, listener);+ },+ } as unknown as Page;+ const secret = "a/b&c\n d";+ const scenario = browserScenarioSchema.parse({+ browser: { mode: "launch", executable_path: "/opt/chromium" },+ start_url: { url: "https://app.example.test/" },+ actions: [+ { step_id: "wait", action: "wait_for_timeout", duration_ms: 1 },+ ],+ secrets: [+ { secret_id: "login_input", environment_variable: "REA_TEST_TOKEN" },+ ],+ capture: { events: ["console", "page-errors", "websockets"] },+ });+ const secrets = BrowserScenarioSecrets.resolve(scenario, {+ REA_TEST_TOKEN: secret,+ });+ if (secrets === undefined) throw new Error("Expected resolved secrets");+ const events = new PlaywrightScenarioEvents({+ page,+ enabled: new Set(["console", "page-errors", "websockets"]),+ secrets,+ });++ listeners.get("console")?.({+ type: () => "log",+ text: () => encodeURIComponent(secret),+ location: () => ({ url: "https://app.example.test/app.js" }),+ });+ const escapedSecret = JSON.stringify(secret).slice(1, -1);+ const error = new Error(`failure ${escapedSecret}`);+ error.stack = `failure ${escapedSecret}`;+ listeners.get("pageerror")?.(error);+ listeners.get("websocket")?.({+ url: () => "wss://app.example.test/socket",+ on: (name: string, listener: (value: unknown) => void) => {+ socketListeners.set(name, listener);+ },+ });+ const formEncodedSecret = new URLSearchParams([["value", secret]])+ .toString()+ .slice("value=".length);+ socketListeners.get("framesent")?.({ payload: formEncodedSecret });++ const items = events.result().items;+ expect(items).toMatchObject([+ { kind: "console", text: "[REDACTED:login_input]" },+ {+ kind: "page-error",+ message: "failure [REDACTED:login_input]",+ stack: "failure [REDACTED:login_input]",+ },+ { kind: "websocket-opened" },+ {+ kind: "websocket-frame-sent",+ payload_text: "[REDACTED:login_input]",+ },+ ]);+ expect(secrets.fingerprint(`stored ${encodeURIComponent(secret)}`)).toEqual(+ { value_state: "redacted-secret", value_sha256: null },+ );+ });+});+describe("PlaywrightScenarioEvents", () => {it("observes requested popup page errors independently of popup lifecycle events", () => {const listeners = new Map<string, (value: unknown) => void>();@@ -42,7 +113,7 @@ describe("PlaywrightScenarioEvents", () => {]);});- it("bounds oversized page errors before validating captured events", () => {+ it("retains oversized page errors without truncating their text", () => {const listeners = new Map<string, (value: Error) => void>();const page = {url: () => "https://app.example.test/",@@ -88,7 +159,7 @@ describe("PlaywrightScenarioEvents", () => {const error = new Error("m".repeat(70_000));error.stack = "s".repeat(300_000);- expect(() => listeners.get("pageerror")?.(error)).not.toThrow();+ listeners.get("pageerror")?.(error);expect(events.result().items[0]).toMatchObject({kind: "page-error",message: "m".repeat(70_000),