Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(javascript): keep encoded URL slashes inside one runtime path segment (#6259)
Kaoru08221639189f269e
diff --git a/docs/javascript-runtime-reconciliation.md b/docs/javascript-runtime-reconciliation.mdindex 068353661..e3236e66b 100644--- a/docs/javascript-runtime-reconciliation.md+++ b/docs/javascript-runtime-reconciliation.md@@ -57,7 +57,11 @@ are complete Evidence objects, not only evidence IDs:`static_layers` requires exactly one `application` role and permits additional`cache` and `assets` roles. A `file-root` mapping translates a canonical runtimefilesystem root to an artifact-relative prefix. A `url-prefix` mapping does the-same for one exact HTTP(S) origin and path prefix. Mappings cannot add CDP+same for one exact HTTP(S) origin and path prefix. Each URL path segment below+the prefix is percent-decoded separately: an encoded slash (`%2F`) stays inside+its segment and never names a nested artifact, so such a location, like one+with malformed or non-UTF-8 escapes, has no artifact path under that mapping.+An encoded backslash (`%5C`) remains a literal name byte. Mappings cannot add CDPorigins, broaden Electron roots, read files, or authorize source-map access;they are caller-declared inference inputs applied only to already-authorizedlocations.@@ -133,6 +137,8 @@ classification:- `unknown` covers incomplete captures, layers outside runtime scope, andunresolved correspondence. Nodes outside a mapped artifact prefix stayunknown even when another prefix in the same static layer was observed.+ Nodes beneath a mapped artifact prefix also stay unknown when a runtime+ script or worker under that mapping has no artifact path.None of these states proves feature execution, initialization order,reachability, or causality. In particular, `not-observed-in-capture` never meansdiff --git a/src/domain/javascript/javascriptRuntimeLoadState.ts b/src/domain/javascript/javascriptRuntimeLoadState.tsindex a1d064efc..87ba847e8 100644--- a/src/domain/javascript/javascriptRuntimeLoadState.ts+++ b/src/domain/javascript/javascriptRuntimeLoadState.ts@@ -9,7 +9,7 @@ import type { RuntimeReconciliationEntity } from "./javascriptRuntimeReconciliatimport {createStaticRuntimeScope,mappedRuntimePaths,- staticNodeWithinRuntimeScope,+ staticNodeRuntimeScope,} from "./javascriptRuntimeStaticCandidates.js";export interface StaticLoadStateProjection {@@ -93,8 +93,8 @@ const classifyLayer = (? "static-or-runtime-coverage-incomplete": "ambiguous-runtime-candidate",});- const nodeWithinScope = staticNodeWithinRuntimeScope(node, runtimeScope);- if (layerComplete && nodeWithinScope)+ const membership = staticNodeRuntimeScope(node, runtimeScope);+ if (layerComplete && membership === "within")return state(layer, node, {status: "not-observed-in-capture",runtimeNodeIds: [],@@ -103,9 +103,10 @@ const classifyLayer = (return state(layer, node, {status: "unknown",runtimeNodeIds: [],- reason: !nodeWithinScope- ? "layer-outside-runtime-scope"- : "static-or-runtime-coverage-incomplete",+ reason:+ membership === "outside"+ ? "layer-outside-runtime-scope"+ : "static-or-runtime-coverage-incomplete",});});return { states };diff --git a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts b/src/domain/javascript/javascriptRuntimeStaticCandidates.tsindex 5c3ef04f6..a870a8fe7 100644--- a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts+++ b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts@@ -42,9 +42,14 @@ export interface MappedRuntimePath {/** Precomputed mapping scope and graph paths for static load-state decisions. */export interface StaticRuntimeScope {readonly artifactPrefixes: readonly string[];+ /** Prefixes beneath which a runtime script or worker had no artifact path. */+ readonly untranslatedArtifactPrefixes: readonly string[];readonly pathsByNodeId: ReadonlyMap<string, readonly string[]>;}+/** Where one static node falls relative to the runtime-exercised mappings. */+export type StaticRuntimeScopeMembership = "outside" | "within" | "uncertain";+/** Index statically reconstructed entities that can receive runtime matches. */export const collectStaticRuntimeCandidates = (layers: readonly ParsedStaticLayer[],@@ -79,13 +84,9 @@ export const mappedRuntimePaths = (});}for (const mapping of layer.runtimeMappings) {- const suffix =- mapping.kind === "file-root" && entity.location.kind === "file"- ? pathBelowRoot(mapping.root, entity.location.value)- : mapping.kind === "url-prefix" && entity.location.kind === "url"- ? pathBelowUrlPrefix(mapping.prefix, entity.location.value)- : null;- if (suffix === null) continue;+ const translation = translateRuntimeLocation(mapping, entity.location);+ if (translation.kind !== "translated") continue;+ const suffix = translation.suffix;const path = safeArtifactPath(mapping.artifact_prefix, suffix);if (path === null) continue;paths.push({@@ -107,18 +108,27 @@ export const createStaticRuntimeScope = (layer: ParsedStaticLayer,entities: readonly RuntimeReconciliationEntity[],): StaticRuntimeScope => ({- artifactPrefixes: activeArtifactPrefixes(layer, entities),+ ...exercisedArtifactPrefixes(layer, entities),pathsByNodeId: indexStaticPaths(layer.graph),});-/** Whether one static node falls under a mapping exercised by this runtime set. */-export const staticNodeWithinRuntimeScope = (+/**+ * Classify one static node against the mappings exercised by this runtime set.+ * A node beneath a prefix whose runtime script or worker location could not be+ * translated is uncertain: that location may name it, so absence is unproven.+ */+export const staticNodeRuntimeScope = (node: ApplicationNode,scope: StaticRuntimeScope,-): boolean =>- (scope.pathsByNodeId.get(node.node_id) ?? []).some((path) =>- scope.artifactPrefixes.some((prefix) => pathWithinPrefix(path, prefix)),- );+): StaticRuntimeScopeMembership => {+ const paths = scope.pathsByNodeId.get(node.node_id) ?? [];+ const beneath = (prefixes: readonly string[]): boolean =>+ paths.some((path) =>+ prefixes.some((prefix) => pathWithinPrefix(path, prefix)),+ );+ if (beneath(scope.untranslatedArtifactPrefixes)) return "uncertain";+ return beneath(scope.artifactPrefixes) ? "within" : "outside";+};const candidatesForLayer = (layer: ParsedStaticLayer,@@ -391,38 +401,87 @@ const absoluteFilePathStyle = (path: string): "posix" | "windows" | null => {return path.startsWith("/") ? "posix" : null;};-const pathBelowUrlPrefix = (prefix: string, value: string): string | null => {- let base: URL;- let candidate: URL;- try {- base = new URL(prefix);- candidate = new URL(value);- } catch (cause: unknown) {- // Invalid input is represented by the null return.- void cause;- return null;- }+/**+ * How a runtime location relates to one mapping: outside it, beneath it with+ * an artifact-relative suffix, or beneath it without any artifact path.+ */+type RuntimeLocationTranslation =+ | { readonly kind: "outside" }+ | { readonly kind: "untranslatable" }+ | { readonly kind: "translated"; readonly suffix: string };++const OUTSIDE_MAPPING: RuntimeLocationTranslation = { kind: "outside" };++const translateRuntimeLocation = (+ mapping: ParsedStaticLayer["runtimeMappings"][number],+ location: RuntimeReconciliationEntity["location"],+): RuntimeLocationTranslation => {+ if (mapping.kind === "url-prefix")+ return location.kind === "url"+ ? pathBelowUrlPrefix(mapping.prefix, location.value)+ : OUTSIDE_MAPPING;+ const suffix =+ location.kind === "file"+ ? pathBelowRoot(mapping.root, location.value)+ : null;+ return suffix === null ? OUTSIDE_MAPPING : { kind: "translated", suffix };+};++const pathBelowUrlPrefix = (+ prefix: string,+ value: string,+): RuntimeLocationTranslation => {+ const base = URL.parse(prefix);+ const candidate = URL.parse(value);if (+ base === null ||+ candidate === null ||base.origin !== candidate.origin ||!candidate.pathname.startsWith(base.pathname))- return null;- try {- return relativeLogicalPath(- decodeURIComponent(candidate.pathname.slice(base.pathname.length)),- );- } catch (cause: unknown) {- // Undecodable path segments are represented by the null return.- void cause;- return null;+ return OUTSIDE_MAPPING;+ const encoded = candidate.pathname.slice(base.pathname.length);+ // The prefix itself names the mapped directory or a page, not a file below it.+ if (encoded === "") return OUTSIDE_MAPPING;+ const decoded = decodeUrlPathSegments(encoded);+ const suffix = decoded === null ? null : relativeLogicalPath(decoded);+ return suffix === null+ ? { kind: "untranslatable" }+ : { kind: "translated", suffix };+};++/**+ * Decode each URL path segment separately. An encoded slash is a byte inside+ * one segment, never an artifact directory boundary, so a segment containing+ * one after decoding has no artifact path. An encoded backslash stays a literal+ * name byte, and malformed or non-UTF-8 escapes have no artifact path.+ */+const decodeUrlPathSegments = (value: string): string | null => {+ const segments: string[] = [];+ for (const segment of value.split("/")) {+ let decoded: string;+ try {+ decoded = decodeURIComponent(segment);+ } catch (cause: unknown) {+ // Malformed escapes are represented by the null return.+ void cause;+ return null;+ }+ if (decoded.includes("/")) return null;+ segments.push(decoded);}+ return segments.join("/");};-const activeArtifactPrefixes = (+const exercisedArtifactPrefixes = (layer: ParsedStaticLayer,entities: readonly RuntimeReconciliationEntity[],-): string[] => {+): Pick<+ StaticRuntimeScope,+ "artifactPrefixes" | "untranslatedArtifactPrefixes"+> => {const prefixes = new Set<string>();+ const untranslated = new Set<string>();if (layer.result.format === "directory" &&entities.some(@@ -432,18 +491,26 @@ const activeArtifactPrefixes = ())prefixes.add("");- for (const mapping of layer.runtimeMappings)- if (- entities.some(({ location }) =>- mapping.kind === "file-root" && location.kind === "file"- ? pathBelowRoot(mapping.root, location.value) !== null- : mapping.kind === "url-prefix" && location.kind === "url"- ? pathBelowUrlPrefix(mapping.prefix, location.value) !== null- : false,+ for (const mapping of layer.runtimeMappings) {+ const prefix = normalizeArtifactPrefix(mapping.artifact_prefix);+ for (const entity of entities) {+ if (prefixes.has(prefix) && untranslated.has(prefix)) break;+ const translation = translateRuntimeLocation(mapping, entity.location);+ if (translation.kind === "translated") prefixes.add(prefix);+ // Only scripts and workers can correspond to load-state nodes.+ else if (+ translation.kind === "untranslatable" &&+ (entity.kind === "script" || entity.kind === "worker"))- )- prefixes.add(normalizeArtifactPrefix(mapping.artifact_prefix));- return [...prefixes].sort(compareUnicodeCodePoints);+ untranslated.add(prefix);+ }+ }+ return {+ artifactPrefixes: [...prefixes].sort(compareUnicodeCodePoints),+ untranslatedArtifactPrefixes: [...untranslated].sort(+ compareUnicodeCodePoints,+ ),+ };};const normalizeArtifactPrefix = (prefix: string): string =>diff --git a/tests/boundary/cli/runtimeUrlPrefixMappingCli.test.ts b/tests/boundary/cli/runtimeUrlPrefixMappingCli.test.tsnew file mode 100644index 000000000..c09e6ceda--- /dev/null+++ b/tests/boundary/cli/runtimeUrlPrefixMappingCli.test.ts@@ -0,0 +1,279 @@+import { execFile } from "node:child_process";+import { mkdir, rm, writeFile } from "node:fs/promises";+import { join } from "node:path";+import { promisify } from "node:util";++import { afterEach, expect, it } from "vitest";+import { z } from "zod";++import { createBrowserEvidence } from "../../../src/application/BrowserEvidence.js";+import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";+import { analyzeFixture } from "../../fixtures/javascriptRuntimeReconciliation.js";+import { passiveNetworkCapture } from "../../fixtures/passiveNetworkCapture.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";++const execute = promisify(execFile);+const INTEGRATION_TEST_TIMEOUT_MS = 30_000;+const temporary: string[] = [];++afterEach(async () => {+ await Promise.all(+ temporary+ .splice(0)+ .map(async (path) => rm(path, { recursive: true, force: true })),+ );+});++const reconciledSchema = z.object({+ normalized_result: z.object({+ reconciliations: z.array(+ z.object({+ entity_kind: z.string(),+ runtime_node_id: z.string(),+ static_node_id: z.string().nullable(),+ status: z.string(),+ basis: z.string(),+ confidence: z.string(),+ reason: z.string(),+ }),+ ),+ static_load_states: z.array(+ z.object({+ static_node_id: z.string(),+ kind: z.string(),+ status: z.string(),+ reason: z.string(),+ }),+ ),+ graph: z.object({+ nodes: z.array(+ z.object({+ node_id: z.string(),+ observations: z.array(+ z.object({+ properties: z.record(z.string(), z.unknown()),+ }),+ ),+ }),+ ),+ edges: z.array(+ z.object({+ relation: z.string(),+ source_node_id: z.string(),+ target_node_id: z.string(),+ }),+ ),+ }),+ }),+});++/** Reconcile one source-free browser capture under `${origin}/assets/`. */+const reconcileThroughCli = async (+ root: string,+ analysis: Awaited<ReturnType<typeof analyzeFixture>>,+ suffixes: readonly string[],+) => {+ const inspection = passiveNetworkCapture([]);+ const origin = inspection.target.origin;+ inspection.frames = [+ { frame_id: "frame-main", parent_frame_id: null, url: origin, origin },+ ];+ inspection.scripts = {+ total: suffixes.length,+ items: suffixes.map((suffix, index) => ({+ script_key: `scr_${String(index).padStart(64, "0")}`,+ frame_id: "frame-main",+ url: `${origin}/assets/${suffix}`,+ origin,+ cdp_hash: null,+ length: null,+ is_module: true,+ language: "JavaScript",+ source_map_url: null,+ resource_reconciliation: {+ status: "unmatched",+ reason: "no_exact_sanitized_url",+ },+ source: { included: false, reason: "not-requested" },+ })),+ };+ const runtime = createBrowserEvidence(+ "inspect_web_page",+ inspectWebPageInputSchema.parse({+ cdp_endpoint: "http://127.0.0.1:9222",+ target_id: "page",+ allowed_origins: [origin],+ observation_ms: 0,+ include_script_sources: false,+ }),+ inspection,+ {+ id: "rea-cdp-browser",+ name: "REA Chrome DevTools Protocol observation provider",+ version: "2",+ },+ );+ const inputPath = join(root, "reconciliation.json");+ await writeFile(+ inputPath,+ JSON.stringify({+ static_layers: [+ {+ role: "application",+ analysis,+ runtime_mappings: [+ {+ kind: "url-prefix",+ prefix: `${origin}/assets/`,+ artifact_prefix: "",+ },+ ],+ },+ ],+ runtime_observations: [runtime],+ }),+ );+ const { stdout, stderr } = await execute(+ process.execPath,+ ["scripts/rea.mjs", "reconcile-javascript-runtime", inputPath, "--json"],+ { cwd: process.cwd(), maxBuffer: 16 * 1_024 * 1_024 },+ );+ expect(stderr).toBe("");+ const result = reconciledSchema.parse(JSON.parse(stdout)).normalized_result;+ const nodes = new Map(result.graph.nodes.map((node) => [node.node_id, node]));+ const property = (nodeId: string | null, key: string) =>+ nodeId === null+ ? null+ : (nodes+ .get(nodeId)+ ?.observations.map(({ properties }) => properties[key])+ .find((value) => typeof value === "string") ?? null);+ const pathOf = (nodeId: string | null) => property(nodeId, "path");+ const scripts = new Map(+ result.reconciliations+ .filter(({ entity_kind }) => entity_kind === "script")+ .map((item) => [+ String(property(item.runtime_node_id, "location")).slice(+ `${origin}/assets/`.length,+ ),+ {+ status: item.status,+ basis: item.basis,+ reason: item.reason,+ path: pathOf(item.static_node_id),+ observed_as: result.graph.edges.filter(+ ({ relation, source_node_id, target_node_id }) =>+ relation === "observed_as" &&+ [source_node_id, target_node_id].includes(item.runtime_node_id),+ ).length,+ },+ ]),+ );+ const loadStates = new Map(+ result.static_load_states+ .filter(({ kind }) => kind === "javascript-asset")+ .map((state) => [+ pathOf(state.static_node_id),+ { status: state.status, reason: state.reason },+ ]),+ );+ return { scripts, loadStates };+};++const unmappedLocation = {+ status: "unknown",+ basis: "none",+ reason: "no-authorized-location-mapping",+ path: null,+ observed_as: 0,+};++const mappedLocation = (path: string) => ({+ status: "matched",+ basis: "operator-url-mapping",+ reason: "unique-location-match",+ path,+ observed_as: 1,+});++it(+ "keeps an encoded slash inside one URL segment rather than naming a nested artifact",+ async () => {+ const root = await createTestTempDirectory("rea-url-prefix-segments-");+ temporary.push(root);+ await mkdir(join(root, "child"));+ await Promise.all([+ writeFile(join(root, "child", "part.js"), "export const nested = 1;\n"),+ writeFile(join(root, "child%2Fpart.js"), "export const literal = 2;\n"),+ writeFile(join(root, "sp ace.js"), "export const space = 3;\n"),+ writeFile(join(root, "é.js"), "export const accent = 4;\n"),+ ]);+ const analysis = await analyzeFixture(root);++ const { scripts } = await reconcileThroughCli(root, analysis, [+ "child/part.js",+ "child/part.js?v=%2F#f%2F",+ "child%2Fpart.js",+ "child%2fpart.js",+ "child%252Fpart.js",+ "sp%20ace.js",+ "%C3%A9.js",+ "child/%E0.js",+ ]);++ expect(Object.fromEntries(scripts)).toEqual({+ "child/part.js": mappedLocation("child/part.js"),+ "child/part.js?v=%2F#f%2F": mappedLocation("child/part.js"),+ "child%2Fpart.js": unmappedLocation,+ "child%2fpart.js": unmappedLocation,+ "child%252Fpart.js": mappedLocation("child%2Fpart.js"),+ "sp%20ace.js": mappedLocation("sp ace.js"),+ "%C3%A9.js": mappedLocation("é.js"),+ "child/%E0.js": unmappedLocation,+ });+ },+ INTEGRATION_TEST_TIMEOUT_MS,+);++it(+ "keeps static load state unknown beneath a prefix whose runtime script had no artifact path",+ async () => {+ const root = await createTestTempDirectory("rea-url-prefix-load-state-");+ temporary.push(root);+ await mkdir(join(root, "child"));+ await Promise.all([+ writeFile(join(root, "child", "part.js"), "export const nested = 1;\n"),+ writeFile(join(root, "other.js"), "export const other = 2;\n"),+ ]);+ const analysis = await analyzeFixture(root);++ const control = await reconcileThroughCli(root, analysis, ["other.js"]);+ expect(Object.fromEntries(control.loadStates)).toEqual({+ "child/part.js": {+ status: "not-observed-in-capture",+ reason: "not-observed-in-bounded-capture",+ },+ "other.js": {+ status: "loaded",+ reason: "runtime-script-correspondence",+ },+ });++ const encoded = await reconcileThroughCli(root, analysis, [+ "other.js",+ "child%2Fpart.js",+ ]);+ expect(encoded.scripts.get("child%2Fpart.js")).toEqual(unmappedLocation);+ expect(Object.fromEntries(encoded.loadStates)).toEqual({+ "child/part.js": {+ status: "unknown",+ reason: "static-or-runtime-coverage-incomplete",+ },+ "other.js": {+ status: "loaded",+ reason: "runtime-script-correspondence",+ },+ });+ },+ INTEGRATION_TEST_TIMEOUT_MS,+);