Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(browser): compose nested source-map column offsets (#1109)
Rudy Mizrahi Celekli8b08ec0af936
diff --git a/docs/web-source-location.md b/docs/web-source-location.mdindex 5191137cf..dae31c3ad 100644--- a/docs/web-source-location.md+++ b/docs/web-source-location.md@@ -84,6 +84,13 @@ are made.## Verification+Browser bundle collection uses absolute section offsets for nested inline maps.+An enclosing column offset contributes only when the nested section starts on+that enclosing section's first generated line. Parent boundaries still clip+preceding browser mappings when a nested section starts with a gap or has no+child leaves. Raw map text, original source identities and the dedicated decoder+profile are preserved.+Focused tests cover duplicate mappings, indexed offsets/overlap/local indexes,32-bit VLQ overflow, nullable content, UTF-16/CRLF positions, immutable artifactidentity, cancellation, late output closure, cleanup failures, SDK schemas anddiff --git a/scripts/fixtures/browser-source-map-site.mjs b/scripts/fixtures/browser-source-map-site.mjsindex 547f54618..e155ec963 100644--- a/scripts/fixtures/browser-source-map-site.mjs+++ b/scripts/fixtures/browser-source-map-site.mjs@@ -4,7 +4,7 @@ import { access } from "node:fs/promises";import { isAbsolute } from "node:path";/** Generate a real compiler map and serve only source-owned fixture code. */-export async function startBrowserSourceMapSite() {+export async function startBrowserSourceMapSite(options = {}) {const command = process.env.REA_WEB_SOURCE_MAP_COMPILER;if (!command || !isAbsolute(command))throw new Error(@@ -19,19 +19,40 @@ export async function startBrowserSourceMapSite() {const compiled = await compiler.transform(original, {loader: "ts",- sourcefile: "../src/fixture.ts?v=7#original",+ sourcefile: options.sourcefile ?? "../src/fixture.ts?v=7#original",sourcemap: "external",minify: true,format: "iife",target: "es2022",}).finally(() => compiler.stop());+ const prefixes = options.prefixes ?? [];+ const generated =+ prefixes.join("") ++ compiled.code ++ (options.sourceMapAnnotation+ ? "\n//# sourceMappingURL=/maps/app.js.map"+ : "");+ let indexed = JSON.parse(compiled.map);+ for (const prefix of prefixes.toReversed()) {+ const offset = textPosition(prefix, prefix.length);+ indexed = {+ version: 3,+ sections: [+ {+ offset: { line: offset.line - 1, column: offset.column },+ map: indexed,+ },+ ],+ };+ }+ const map = prefixes.length === 0 ? compiled.map : JSON.stringify(indexed);const token = '"source-map proof"';- const generatedOffset = compiled.code.indexOf(token);+ const generatedOffset = generated.indexOf(token);const originalOffset = original.indexOf(token);if (generatedOffset < 0 || originalOffset < 0)throw new Error("Compiler fixture marker missing");- const position = textPosition(compiled.code, generatedOffset);+ const position = textPosition(generated, generatedOffset);const originalPosition = textPosition(original, originalOffset);let mapRequests = 0;const server = createServer((request, response) => {@@ -42,12 +63,12 @@ export async function startBrowserSourceMapSite() {if (request.url === "/app.js?build=7")return response.writeHead(200, { "Content-Type": "text/javascript" })- .end(compiled.code);+ .end(generated);if (request.url?.startsWith("/maps/app.js.map")) {mapRequests += 1;return response.writeHead(200, { "Content-Type": "application/json" })- .end(compiled.map);+ .end(map);}response.writeHead(404).end("Unselected");});@@ -58,8 +79,10 @@ export async function startBrowserSourceMapSite() {return {origin: `http://127.0.0.1:${address.port}`,original,- generated: compiled.code,- map: compiled.map,+ generated,+ map,+ leafCode: compiled.code,+ leafMap: compiled.map,position,originalPosition,originalOffset,diff --git a/scripts/lib/browser-nested-source-map-e2e.mjs b/scripts/lib/browser-nested-source-map-e2e.mjsnew file mode 100644index 000000000..4e76f4258--- /dev/null+++ b/scripts/lib/browser-nested-source-map-e2e.mjs@@ -0,0 +1,238 @@+import assert from "node:assert/strict";+import { execFile } from "node:child_process";+import { createHash } from "node:crypto";+import { join } from "node:path";+import { fileURLToPath } from "node:url";+import { promisify } from "node:util";+import { TraceMap, eachMapping } from "@jridgewell/trace-mapping";+import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js";+import { CdpConnection } from "../../dist/browser/CdpConnection.js";+import { waitForBrowserDevtoolsPort } from "../../dist/browser/BrowserProcessStartup.js";+import { parseEvidence } from "../../dist/domain/evidence.js";+import { traceSourceMap } from "../../dist/javascript/sourceMaps/TraceSourceMap.js";+import { startBrowserSourceMapSite } from "../fixtures/browser-source-map-site.mjs";+import {+ startRuntimeFixtureBrowser,+ closeRuntimeFixtureResources,+} from "./browser-runtime-fixture-lifecycle.mjs";++const cases = [+ { name: "same-line nested column control", prefixes: ["\n\n ", " "] },+ { name: "zero outer column control", prefixes: ["\n\n", "\n "] },+ { name: "nested line resets outer column", prefixes: ["\n\n ", "\n"] },+ {+ name: "nested line retains its own column",+ prefixes: ["\n\n ", "\n "],+ },+];++/** Verify compiler-produced nested maps through an actual owned Chrome and public CLI. */+export async function verifyBrowserNestedSourceMaps(+ executable,+ entrypoint = fileURLToPath(new URL("../rea.mjs", import.meta.url)),+) {+ const results = [];+ for (const scenario of cases) {+ try {+ const proof = await verifyCase(executable, entrypoint, scenario);+ results.push({ name: scenario.name, status: "pass", ...proof });+ } catch (cause) {+ results.push({+ name: scenario.name,+ status: "fail",+ phase: cause.qualificationPhase ?? "native-producer-admission",+ proof: cause.qualificationReceipt ?? null,+ error: String(cause),+ });+ }+ }+ // Preserve every affected owner and control even when BEFORE fails.+ process.stdout.write(+ `${JSON.stringify({ nested_source_map_cases: results })}\n`,+ );+ assert.equal(results.filter(({ status }) => status === "fail").length, 0);+ return { cases: results, actual_chrome: true, public_cli: true };+}++async function verifyCase(executable, entrypoint, scenario) {+ const site = await startBrowserSourceMapSite({+ prefixes: scenario.prefixes,+ sourceMapAnnotation: true,+ sourcefile: "../src/nested-fixture.ts",+ });+ let fixture;+ let connection;+ let primaryError;+ let stderr = "";+ let phase = "native-map-admission";+ let browserVersion;+ let expected;+ let observed;+ try {+ const mapUrl = `${site.origin}/maps/app.js.map`;+ // This independent existing decoder already composes valid nested offsets.+ // It is a before-product admission control, not the expected browser oracle.+ const point = traceSourceMap(site.map, mapUrl, site.position);+ assert.equal(point.matches.length, 1);+ assert.equal(+ point.matches[0].original_position.line,+ site.originalPosition.line,+ );+ assert.equal(+ point.matches[0].original_position.column,+ site.originalPosition.column,+ );+ expected = compilerMappingsAtPhysicalPositions(site, mapUrl);+ assert.ok(+ expected.length > 0,+ "Native compiler leaf must carry point mappings",+ );++ phase = "owned-browser-startup";+ fixture = await startRuntimeFixtureBrowser(executable, site.origin);+ fixture.browser.stderr.on("data", (chunk) => {+ if (stderr.length < 65536) stderr += chunk;+ });+ const port = await waitForBrowserDevtoolsPort({+ child: fixture.browser,+ executable,+ activePortPath: join(fixture.profile, "DevToolsActivePort"),+ stderr: () => stderr,+ timeoutMs: 20_000,+ });+ const endpoint = `http://127.0.0.1:${port}`;+ const provider = new CdpBrowserProvider();+ const targets = await provider.listTargets({+ cdp_endpoint: endpoint,+ allowed_origins: [site.origin],+ });+ if (!targets.ok) throw targets.error;+ const target = targets.value.targets[0];+ assert.ok(target, "Owned fixture target missing");+ const pages = await (await fetch(`${endpoint}/json/list`)).json();+ const page = pages.find(({ id }) => id === target.target_id);+ assert.ok(page, "Owned CDP fixture page missing");+ connection = await CdpConnection.connect(+ page.webSocketDebuggerUrl,+ "inspect_web_page",+ );+ await ready(connection);+ browserVersion = await connection.send("Browser.getVersion");+ phase = "public-cli";+ const { stdout } = await promisify(execFile)(+ process.execPath,+ [+ entrypoint,+ "analyze-web-bundle",+ endpoint,+ target.target_id,+ "--allowed-origins",+ site.origin,+ "--fetch-source-maps",+ "--observation-ms",+ "0",+ "--json",+ ],+ {+ env: { ...process.env, REA_LOG_LEVEL: "silent" },+ timeout: 45_000,+ maxBuffer: 16 * 1024 * 1024,+ },+ );+ const evidence = parseEvidence(JSON.parse(stdout));+ assert.equal(evidence.operation, "analyze_web_bundle");+ const maps = evidence.normalized_result.observations.source_maps;+ const item = maps.items.find(({ artifact }) => artifact?.text === site.map);+ assert.ok(+ item,+ "Public browser analysis omitted the exact compiler-derived map",+ );+ phase = "public-map-assertions";+ observed = item?.mappings;+ assert.equal(item.status, "included");+ assert.deepEqual(item.mappings, expected);+ assert.equal(item.artifact.sha256, sha(site.map));+ assert.equal(item.artifact.bytes, Buffer.byteLength(site.map));+ assert.equal(item.original_sources[0].artifact.sha256, sha(site.original));+ assert.ok(+ site.mapRequests() > 0,+ "Actual browser workflow did not fetch the declared map",+ );+ return {+ compiler: site.compiler,+ browser: browserVersion.product,+ generated_sha256: sha(site.generated),+ source_map_sha256: sha(site.map),+ original_sha256: sha(site.original),+ mappings: expected.length,+ native_decoder_control: true,+ };+ } catch (cause) {+ const failure = new Error(`Nested source-map ${phase}: ${String(cause)}`, {+ cause,+ });+ failure.qualificationPhase = phase;+ failure.qualificationReceipt = {+ compiler: site.compiler,+ browser: browserVersion?.product ?? null,+ generated_sha256: sha(site.generated),+ leaf_map_sha256: sha(site.leafMap),+ source_map_sha256: sha(site.map),+ original_sha256: sha(site.original),+ expected_first_mapping: expected?.[0] ?? null,+ observed_first_mapping: observed?.[0] ?? null,+ };+ primaryError = failure;+ throw failure;+ } finally {+ await closeRuntimeFixtureResources(+ [() => connection?.close(), () => fixture?.close(), () => site.close()],+ primaryError,+ );+ }+}++function compilerMappingsAtPhysicalPositions(site, mapUrl) {+ const expected = [];+ const prefixLength = site.generated.indexOf(site.leafCode);+ assert.ok(prefixLength >= 0);+ eachMapping(new TraceMap(site.leafMap, mapUrl), (mapping) => {+ if (+ mapping.source === null ||+ mapping.originalLine === null ||+ mapping.originalColumn === null+ )+ return;+ const leafLines = site.leafCode.split("\n");+ const before = leafLines.slice(0, mapping.generatedLine - 1).join("\n");+ const leafOffset =+ before.length ++ (mapping.generatedLine > 1 ? 1 : 0) ++ mapping.generatedColumn;+ const physical = site.generated+ .slice(0, prefixLength + leafOffset)+ .split("\n");+ expected.push({+ generated_line: physical.length,+ generated_column: physical.at(-1).length,+ source: mapping.source,+ original_line: mapping.originalLine,+ original_column: mapping.originalColumn,+ name: mapping.name ?? null,+ });+ });+ return expected;+}++async function ready(connection) {+ for (let attempt = 0; attempt < 100; attempt += 1) {+ const result = await connection.send("Runtime.evaluate", {+ expression: 'document.body.dataset.ready === "source-map proof"',+ returnByValue: true,+ });+ if (result.result.value === true) return;+ await new Promise((resolve) => setTimeout(resolve, 20));+ }+ throw new Error("Owned compiler fixture did not execute in actual Chrome");+}+const sha = (text) => createHash("sha256").update(text).digest("hex");diff --git a/scripts/verify-browser-source-maps.mjs b/scripts/verify-browser-source-maps.mjsindex 99cdcb80d..a8b53e504 100644--- a/scripts/verify-browser-source-maps.mjs+++ b/scripts/verify-browser-source-maps.mjs@@ -2,6 +2,7 @@import { access } from "node:fs/promises";import { isAbsolute } from "node:path";import { verifyBrowserSourceMap } from "./lib/browser-source-map-e2e.mjs";+import { verifyBrowserNestedSourceMaps } from "./lib/browser-nested-source-map-e2e.mjs";import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";const executable = process.env.REA_BROWSER_EXECUTABLE;if (!executable || !isAbsolute(executable))@@ -11,6 +12,7 @@ if (!executable || !isAbsolute(executable))await access(executable);const run = createVerifierRun();const proof = await verifyBrowserSourceMap(executable, process.argv[2]);+const nested = await verifyBrowserNestedSourceMaps(executable, process.argv[2]);process.stdout.write(- `${JSON.stringify({ ...proof, verifier_run: await completeVerifierRun(run), verified: true })}\n`,+ `${JSON.stringify({ ...proof, nested_source_maps: nested, verifier_run: await completeVerifierRun(run), verified: true })}\n`,);diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.tsindex 3d41d020f..6adb381f3 100644--- a/src/browser/WebSourceMapFetcher.ts+++ b/src/browser/WebSourceMapFetcher.ts@@ -1,5 +1,11 @@import * as t from "@babel/types";-import { AnyMap, eachMapping } from "@jridgewell/trace-mapping";+import {+ AnyMap,+ eachMapping,+ type EncodedSourceMap,+ type Section,+ type SectionedSourceMap,+} from "@jridgewell/trace-mapping";import { sanitizeBrowserUrl } from "../domain/browserObservation.js";import { isUrlLikeModuleSpecifier } from "../domain/webBundleAnalyzerAst.js";@@ -336,6 +342,47 @@ const promiseWithAbort = (});};+type EncodedBrowserSourceMap = EncodedSourceMap | SectionedSourceMap;++/** Compose indexed offsets before upstream flattening, retaining parent clipping. */+const sourceMapWithAbsoluteSections = (+ text: string,+): EncodedBrowserSourceMap => {+ const root = JSON.parse(text) as EncodedBrowserSourceMap;+ if (!("sections" in root)) return root;+ const sections: Section[] = [];+ const pending: Section[] = [{ map: root, offset: { line: 0, column: 0 } }];+ while (pending.length > 0) {+ const node = pending.pop();+ if (node === undefined) break;+ if (!("sections" in node.map)) {+ sections.push(node);+ continue;+ }+ // A parent's boundary clips the preceding leaf even when its first child+ // starts later or it has no children. Empty maps retain that boundary+ // without adding source/name identities or point mappings.+ sections.push({+ offset: node.offset,+ map: { version: 3, sources: [], names: [], mappings: "" },+ });+ for (let index = node.map.sections.length - 1; index >= 0; index -= 1) {+ const child = node.map.sections[index];+ if (child === undefined) continue;+ pending.push({+ map: child.map,+ offset: {+ line: node.offset.line + child.offset.line,+ column:+ child.offset.column ++ (child.offset.line === 0 ? node.offset.column : 0),+ },+ });+ }+ }+ return { ...root, sections };+};+const normalizeSourceMap = (request: WebSourceMapRequest,text: string,@@ -348,7 +395,7 @@ const normalizeSourceMap = ("Source-map JSON is not a version 3 map.",);try {- const map = new AnyMap(text, fetchedUrl);+ const map = new AnyMap(sourceMapWithAbsoluteSections(text), fetchedUrl);const resolvedBySource = new Map<string, string>();const originalSources = map.sources.map((source, index) => {const content = map.sourcesContent?.[index];diff --git a/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts b/tests/boundary/browser/webSourceMapNestedCoordinates.test.tsnew file mode 100644index 000000000..ac6bc1a1d--- /dev/null+++ b/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts@@ -0,0 +1,360 @@+import { createHash } from "node:crypto";+import { expect, it } from "vitest";+import { fetchWebSourceMaps } from "../../../src/browser/WebSourceMapFetcher.js";+import { analyzeWebBundleInputSchema } from "../../../src/domain/webBundleAnalysis.js";+import { traceSourceMap } from "../../../src/javascript/sourceMaps/TraceSourceMap.js";++const cases = [+ {+ name: "later nested line resets outer column",+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 5,+ column: 10,+ },+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 1,+ column: 0,+ },+ map: {+ version: 3,+ sources: ["a.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA",+ },+ },+ ],+ },+ },+ ],+ },+ expected: [[7, 0, "a.ts"]],+ dedicatedControl: true,+ },+ {+ name: "same line retains summed columns",+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 5,+ column: 10,+ },+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 0,+ column: 3,+ },+ map: {+ version: 3,+ sources: ["a.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA",+ },+ },+ ],+ },+ },+ ],+ },+ expected: [[6, 13, "a.ts"]],+ dedicatedControl: true,+ },+ {+ name: "ordinary leaf unchanged",+ map: {+ version: 3,+ sources: ["a.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA;AACA",+ },+ expected: [+ [1, 0, "a.ts"],+ [2, 0, "a.ts"],+ ],+ dedicatedControl: true,+ },+ {+ name: "same-line sibling clipping",+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 0,+ column: 0,+ },+ map: {+ version: 3,+ sources: ["a.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA,QAAQ,IAAI",+ },+ },+ {+ offset: {+ line: 0,+ column: 10,+ },+ map: {+ version: 3,+ sources: ["b.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA",+ },+ },+ ],+ },+ expected: [+ [1, 0, "a.ts"],+ [1, 8, "a.ts"],+ [1, 10, "b.ts"],+ ],+ dedicatedControl: false,+ },+ {+ name: "nested gap clips previous parent leaf",+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 0,+ column: 0,+ },+ map: {+ version: 3,+ sources: ["a.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA,QAAQ,IAAI",+ },+ },+ {+ offset: {+ line: 0,+ column: 5,+ },+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 0,+ column: 3,+ },+ map: {+ version: 3,+ sources: ["b.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA",+ },+ },+ ],+ },+ },+ ],+ },+ expected: [+ [1, 0, "a.ts"],+ [1, 8, "b.ts"],+ ],+ dedicatedControl: false,+ },+ {+ name: "empty indexed sibling clips previous leaf",+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 0,+ column: 0,+ },+ map: {+ version: 3,+ sources: ["a.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA,QAAQ,IAAI",+ },+ },+ {+ offset: {+ line: 0,+ column: 5,+ },+ map: {+ version: 3,+ sections: [],+ },+ },+ ],+ },+ expected: [[1, 0, "a.ts"]],+ dedicatedControl: false,+ },+ {+ name: "depth-three same-line composition",+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 0,+ column: 2,+ },+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 0,+ column: 3,+ },+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 0,+ column: 4,+ },+ map: {+ version: 3,+ sources: ["a.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA",+ },+ },+ ],+ },+ },+ ],+ },+ },+ ],+ },+ expected: [[1, 9, "a.ts"]],+ dedicatedControl: true,+ },+ {+ name: "multiline inherited sibling stop",+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 0,+ column: 0,+ },+ map: {+ version: 3,+ sections: [+ {+ offset: {+ line: 1,+ column: 3,+ },+ map: {+ version: 3,+ sources: ["a.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA;AACA;AACA;AACA",+ },+ },+ ],+ },+ },+ {+ offset: {+ line: 3,+ column: 1,+ },+ map: {+ version: 3,+ sources: ["b.ts"],+ sourcesContent: ["export const value=1;"],+ names: [],+ mappings: "AAAA",+ },+ },+ ],+ },+ expected: [+ [2, 3, "a.ts"],+ [3, 0, "a.ts"],+ [4, 0, "a.ts"],+ [4, 1, "b.ts"],+ ],+ dedicatedControl: false,+ },+] as const;++it.each(cases)(+ "preserves indexed coordinate contract: $name",+ async ({ map, expected, dedicatedControl }) => {+ const origin = "https://example.test";+ const url = origin + "/app.js.map";+ const text = JSON.stringify(map);+ // The dedicated decoder already normalizes absolute leaves. Its stricter+ // overlap profile is compared only for the valid non-overlapping maps.+ if (dedicatedControl) {+ for (const [line, column] of expected) {+ const point = traceSourceMap(text, url, { line, column });+ expect(point.lookup.matched_generated_position).toEqual({+ line,+ column,+ });+ expect(point.matches).toHaveLength(1);+ }+ }+ const result = await fetchWebSourceMaps(+ [{ scriptKey: "scr_" + "1".repeat(64), declaredUrl: url, fetchUrl: url }],+ analyzeWebBundleInputSchema.parse({+ cdp_endpoint: "http://127.0.0.1:9222",+ allowed_origins: [origin],+ target_id: "page-1",+ fetch_source_maps: true,+ }),+ undefined,+ { fetch: () => Promise.resolve(new Response(text, { status: 200 })) },+ );+ const item = result.items[0];+ if (item?.status !== "included") throw new Error(JSON.stringify(result));+ expect(+ item.mappings.map((entry) => [+ entry.generated_line,+ entry.generated_column,+ entry.source.replace(origin + "/", ""),+ ]),+ ).toEqual(expected);+ expect(item.artifact.sha256).toBe(+ createHash("sha256").update(text).digest("hex"),+ );+ expect(item.artifact.bytes).toBe(Buffer.byteLength(text));+ // Boundary-only markers must not introduce empty original-source identities.+ expect(item.original_sources.map(({ source }) => source)).toEqual([+ ...new Set(expected.map(([, , source]) => origin + "/" + source)),+ ]);+ },+);