Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(browser): preserve URL paths when removing credentials (#4731)
ShuaixinHuang6b0113a534aa
diff --git a/scripts/lib/browser-url-userinfo-e2e.mjs b/scripts/lib/browser-url-userinfo-e2e.mjsnew file mode 100644index 000000000..4506c0445--- /dev/null+++ b/scripts/lib/browser-url-userinfo-e2e.mjs@@ -0,0 +1,186 @@+import assert from "node:assert/strict";+import { createHash } from "node:crypto";+import { fileURLToPath } from "node:url";+import { Client } from "@modelcontextprotocol/client";+import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";++import { CdpConnection } from "../../dist/browser/CdpConnection.js";+import { parseEvidence } from "../../dist/domain/evidence.js";+import { artifactCliEvidence } from "./artifact-e2e.mjs";+import { closeVerifierResources } from "./verifier-cleanup.mjs";++/** Verify real V8 source labels, which retain spelling that navigation normalizes. */+export async function verifyBrowserUrlUserInfo(endpoint, origin) {+ const rawUrl = `${origin.replace("://", "://user:pass@")}\\path@tail/../keep.js?x=1#frag`;+ const expectedUrl = `${origin}\\path@tail/../keep.js?x=1#frag`;+ const sources = [rawUrl, expectedUrl].map(+ (url, index) =>+ `globalThis.reaUserinfoUrlProbe${index} = () => 4731;\n//# sourceURL=${url}`,+ );+ const response = await fetch(`${endpoint}/json/new?about:blank`, {+ method: "PUT",+ });+ assert.equal(+ response.ok,+ true,+ "Browser refused the owned URL fixture target",+ );+ const target = await response.json();+ assert.equal(typeof target.id, "string");+ let connection;+ let transport;+ let client;+ let verificationError;+ try {+ connection = await CdpConnection.connect(+ target.webSocketDebuggerUrl,+ "inspect_web_page",+ );+ await loadFixturePage(connection, `${origin}/app?userinfo-probe`);+ const producerUrls = [];+ connection.onEvent((event) => {+ if (event.method === "Debugger.scriptParsed")+ producerUrls.push(event.params.url);+ });+ await connection.send("Debugger.enable");+ const browserUrl = await connection.send("Runtime.evaluate", {+ expression: `new URL(${JSON.stringify(rawUrl)}).href`,+ returnByValue: true,+ });+ assert.equal(browserUrl.exceptionDetails, undefined);+ assert.equal(browserUrl.result.value, new URL(rawUrl).href);+ assert.notEqual(browserUrl.result.value, rawUrl);+ assert.equal(new URL(browserUrl.result.value).pathname, "/keep.js");+ for (const source of sources) {+ const evaluated = await connection.send("Runtime.evaluate", {+ expression: source,+ returnByValue: true,+ });+ assert.equal(evaluated.exceptionDetails, undefined);+ }+ assert.ok(+ [rawUrl, expectedUrl].every((url) => producerUrls.includes(url)),+ "Real V8 did not retain the exact sourceURL spelling on the CDP wire",+ );+ const assertScripts = (evidence) => {+ const result = parseEvidence(evidence).normalized_result;+ return sources.map((source) => {+ const script = result.scripts.items.find(+ (item) =>+ item.source.included && item.source.artifact.text === source,+ );+ assert.ok(+ script,+ "Public capture omitted the retained V8 fixture script",+ );+ assert.equal(script.url, expectedUrl);+ assert.equal(new URL(script.url).origin, origin);+ assert.equal(+ script.source.artifact.sha256,+ createHash("sha256").update(source).digest("hex"),+ );+ return script.url;+ });+ };+ const cliUrl = assertScripts(+ await artifactCliEvidence("inspect-web-page", endpoint, [+ target.id,+ "--allowed-origins",+ origin,+ "--include-script-sources",+ "--observation-ms",+ "0",+ ]),+ );+ transport = new StdioClientTransport({+ command: process.execPath,+ args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"],+ env: { ...process.env, REA_LOG_LEVEL: "silent" },+ stderr: "pipe",+ });+ client = new Client({ name: "browser-url-userinfo-e2e", version: "1" });+ await client.connect(transport);+ const mcp = await client.callTool({+ name: "inspect_web_page",+ arguments: {+ cdp_endpoint: endpoint,+ allowed_origins: [origin],+ target_id: target.id,+ include_script_sources: true,+ observation_ms: 0,+ },+ });+ assert.notEqual(mcp.isError, true, JSON.stringify(mcp));+ assert.deepEqual(assertScripts(mcp.structuredContent), cliUrl);+ return {+ cli: true,+ stdio_mcp: true,+ raw_v8_source_url: true,+ credential_free_control: true,+ original_source_digest: true,+ browser_url_parser_normalizes_backslash_and_dot_segments: true,+ };+ } catch (cause) {+ verificationError = cause;+ throw cause;+ } finally {+ await closeVerifierResources(+ [+ async () => client?.close(),+ async () => transport?.close(),+ async () => connection?.close(),+ async () => {+ const closed = await fetch(`${endpoint}/json/close/${target.id}`);+ assert.equal(+ closed.ok,+ true,+ "Browser retained the owned URL fixture target",+ );+ },+ ],+ verificationError,+ );+ }+}++async function loadFixturePage(connection, url) {+ const loaded = new Set();+ let loaderId;+ let resolveLoaded;+ let timer;+ const load = new Promise((resolve) => {+ resolveLoaded = resolve;+ });+ const unsubscribe = connection.onEvent((event) => {+ if (event.method !== "Page.lifecycleEvent" || event.params.name !== "load")+ return;+ loaded.add(event.params.loaderId);+ if (event.params.loaderId === loaderId) resolveLoaded();+ });+ try {+ await connection.send("Page.enable");+ await connection.send("Page.setLifecycleEventsEnabled", { enabled: true });+ const navigation = await connection.send("Page.navigate", { url });+ assert.equal(navigation.errorText, undefined);+ assert.equal(typeof navigation.loaderId, "string");+ loaderId = navigation.loaderId;+ if (!loaded.has(loaderId))+ await Promise.race([+ load,+ new Promise((_, reject) => {+ timer = setTimeout(+ () => reject(new Error("Owned URL fixture did not finish loading")),+ 10_000,+ );+ }),+ ]);+ assert.equal(+ (await connection.send("Page.getFrameTree")).frameTree.frame.url,+ url,+ );+ } finally {+ clearTimeout(timer);+ unsubscribe();+ await connection.send("Page.setLifecycleEventsEnabled", { enabled: false });+ }+}diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjsindex f412aafdd..54c1eaa11 100644--- a/scripts/verify-real-browser.mjs+++ b/scripts/verify-real-browser.mjs@@ -53,6 +53,7 @@ import { artifactCliEvidence, artifactMcpResult } from "./lib/artifact-e2e.mjs";import { verifyScenarioEnvironment } from "./lib/browser-scenario-environment-e2e.mjs";import { verifyScenarioStorage } from "./lib/browser-scenario-storage-e2e.mjs";import { verifyScenarioServiceWorkers } from "./lib/browser-scenario-service-workers-e2e.mjs";+import { verifyBrowserUrlUserInfo } from "./lib/browser-url-userinfo-e2e.mjs";const SCENARIO_SECRET_VALUE = "rea-browser-verifier-secret";const SCENARIO_URL_SECRET_VALUE = "rea-browser-url-verifier-secret";@@ -378,6 +379,7 @@ try {});const moduleTrace = await verifyBrowserModules(executable);const captureMetadata = await verifyBrowserCaptureMetadataBudget(endpoint);+ const urlUserInfo = await verifyBrowserUrlUserInfo(endpoint, site.origin);report = {browser: observed.value.browser.product,endpoint,@@ -405,6 +407,7 @@ try {scriptExport,moduleTrace,captureMetadata,+ urlUserInfo,domDestinations,browserScenarioCli: true,browserScenarioAttachCleanup: "disconnected-external",diff --git a/src/domain/browserObservation.test.ts b/src/domain/browserObservation.test.tsindex 6bcf09f68..2de2a7cd2 100644--- a/src/domain/browserObservation.test.ts+++ b/src/domain/browserObservation.test.ts@@ -123,6 +123,42 @@ describe("browser observation sensitive surfaces and retention", () => {expect(sanitized.redacted).toBe(true);});+ it("preserves path userinfo-like text after a special-scheme backslash", () => {+ const suffix = String.raw`\path@tail/../keep?x=1#frag`;+ expect(+ sanitizeBrowserUrl(`https://user:[email protected]${suffix}`),+ ).toEqual({+ url: `https://app.example.test${suffix}`,+ origin: "https://app.example.test",+ query_parameter_names: ["x"],+ redacted: true,+ });+ expect(sanitizeBrowserUrl(`https://app.example.test${suffix}`)).toEqual({+ url: `https://app.example.test${suffix}`,+ origin: "https://app.example.test",+ query_parameter_names: ["x"],+ redacted: false,+ });+ expect(+ sanitizeBrowserUrl(+ String.raw`custom://user:pass@host\name@tail/path?x=1#frag`,+ ),+ ).toEqual({+ url: "custom://tail/path?x=1#frag",+ origin: null,+ query_parameter_names: ["x"],+ redacted: true,+ });+ expect(+ sanitizeBrowserUrl("https://us\ter:[email protected]/path?x=1#frag"),+ ).toEqual({+ url: "https://app.example.test/path?x=1#frag",+ origin: "https://app.example.test",+ query_parameter_names: ["x"],+ redacted: true,+ });+ });+it("retains ordinary query values and malformed URL diagnostics", () => {const longName = `a${"x".repeat(400)}`;const parameters = [diff --git a/src/domain/browserObservation.ts b/src/domain/browserObservation.tsindex a1e669b26..0c513ecee 100644--- a/src/domain/browserObservation.ts+++ b/src/domain/browserObservation.ts@@ -1,7 +1,10 @@import { z } from "zod";import { urlDeclaresQueryOrFragment } from "./urlComponents.js";-import { parseUrlQueryParameterNames } from "./recordedTransportUrl.js";+import {+ parseRecordedTransportUrl,+ parseUrlQueryParameterNames,+} from "./recordedTransportUrl.js";const parseExactOrigin = (value: string): string | undefined => {let url: URL;@@ -268,32 +271,10 @@ export const sanitizeBrowserUrl = (value: string): SanitizedBrowserUrl => {};const removeUrlUserInfo = (value: string, parsed: URL): string => {- const schemeSeparator = value.indexOf("://");- const authorityStart =- schemeSeparator < 0- ? value.startsWith("//")- ? 2- : -1- : schemeSeparator + 3;- if (authorityStart < 0) return credentialFreeHref(parsed);- const remaining = value.slice(authorityStart);- const delimiter = remaining.search(/[/?#]/u);- const authorityEnd =- delimiter < 0 ? value.length : authorityStart + delimiter;- const authority = value.slice(authorityStart, authorityEnd);- const userInfoEnd = authority.lastIndexOf("@");- if (userInfoEnd < 0) return credentialFreeHref(parsed);- const candidate = `${value.slice(0, authorityStart)}${authority.slice(userInfoEnd + 1)}${value.slice(authorityEnd)}`;- try {- const candidateUrl = new URL(candidate);- return candidateUrl.username === "" && candidateUrl.password === ""- ? candidate- : credentialFreeHref(parsed);- } catch (cause: unknown) {- // Fallback to the already-sanitized href when the candidate is invalid.- void cause;- return credentialFreeHref(parsed);- }+ const { credentials } = parseRecordedTransportUrl(value);+ return credentials === null || credentials === "ambiguous"+ ? credentialFreeHref(parsed)+ : `${value.slice(0, credentials.start)}${value.slice(credentials.end)}`;};const credentialFreeHref = (parsed: URL): string => {diff --git a/tests/boundary/browser/cdpBrowserProvider.targetInventory.test.ts b/tests/boundary/browser/cdpBrowserProvider.targetInventory.test.tsindex 9b2053d39..9f3c3153a 100644--- a/tests/boundary/browser/cdpBrowserProvider.targetInventory.test.ts+++ b/tests/boundary/browser/cdpBrowserProvider.targetInventory.test.ts@@ -2,11 +2,12 @@ import { createServer } from "node:http";import { describe, expect, it } from "vitest";-import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js";-import { listBrowserTargetsInputSchema } from "../../../src/domain/browserObservation.js";+import { parseEvidence } from "../../../src/domain/evidence.js";+import { browserTargetListSchema } from "../../../src/domain/browserObservationSchemas.js";+import { runCliJson } from "../../fixtures/cliJsonProcess.js";describe("CDP target inventory", () => {- it("returns every allowed target within the bounded discovery payload", async () => {+ it("returns complete target inventory and original URL paths through the CLI", async () => {let port = 0;const targets = Array.from({ length: 1_101 }, (_, index) => {const id = `target-${String(index).padStart(4, "0")}`;@@ -14,7 +15,10 @@ describe("CDP target inventory", () => {id,type: "page",title: `Page ${String(index)}`,- url: `https://allowed.example/page/${String(index)}`,+ url:+ index === 0+ ? String.raw`https://user:[email protected]\path@tail/../keep?x=1#frag`+ : `https://allowed.example/page/${String(index)}`,...(index === 0 ? {} : { attached: index === 1 ? false : true }),webSocketDebuggerUrl: `ws://127.0.0.1:${String(port)}/devtools/page/${id}`,};@@ -52,24 +56,30 @@ describe("CDP target inventory", () => {port = address.port;const endpoint = `http://127.0.0.1:${String(port)}`;- const result = await new CdpBrowserProvider().listTargets(- listBrowserTargetsInputSchema.parse({- cdp_endpoint: endpoint,- allowed_origins: ["https://allowed.example"],- }),+ const evidence = parseEvidence(+ await runCliJson([+ "list-browser-targets",+ endpoint,+ "--allowed-origins",+ "https://allowed.example",+ "--json",+ ]),);+ const result = browserTargetListSchema.parse(evidence.normalized_result);- if (!result.ok) throw result.error;- expect(result.value.targets).toHaveLength(targets.length);- expect(result.value.targets[0]?.target_id).toBe("target-0000");- expect(result.value.targets.at(-1)?.target_id).toBe("target-1100");+ expect(result.targets).toHaveLength(targets.length);+ expect(result.targets[0]?.target_id).toBe("target-0000");+ expect(result.targets[0]?.url).toBe(+ String.raw`https://allowed.example\path@tail/../keep?x=1#frag`,+ );+ expect(result.targets[0]?.origin).toBe("https://allowed.example");+ expect(JSON.stringify(evidence)).not.toContain("user:pass");+ expect(result.targets.at(-1)?.target_id).toBe("target-1100");expect(- result.value.targets.slice(0, 3).map(({ attached }) => attached),+ result.targets.slice(0, 3).map(({ attached }) => attached),).toEqual([null, false, true]);expect(- result.value.limitations.some((item) =>- item.includes("attachment state"),- ),+ result.limitations.some((item) => item.includes("attachment state")),).toBe(true);} finally {if (server.listening)