Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(javascript): keep encoded URL slashes inside one runtime path segment (#6259)

Kaoru08221639189f269e
diff --git a/docs/javascript-runtime-reconciliation.md b/docs/javascript-runtime-reconciliation.md
index 068353661..e3236e66b 100644
--- a/docs/javascript-runtime-reconciliation.md
+++ b/docs/javascript-runtime-reconciliation.md
@@ -57,7 +57,11 @@ are complete Evidence objects, not only evidence IDs:
`static_layers` requires exactly one `application` role and permits additional
`cache` and `assets` roles. A `file-root` mapping translates a canonical runtime
filesystem root to an artifact-relative prefix. A `url-prefix` mapping does the
-same for one exact HTTP(S) origin and path prefix. Mappings cannot add CDP
+same for one exact HTTP(S) origin and path prefix. Each URL path segment below
+the prefix is percent-decoded separately: an encoded slash (`%2F`) stays inside
+its segment and never names a nested artifact, so such a location, like one
+with malformed or non-UTF-8 escapes, has no artifact path under that mapping.
+An encoded backslash (`%5C`) remains a literal name byte. Mappings cannot add CDP
origins, broaden Electron roots, read files, or authorize source-map access;
they are caller-declared inference inputs applied only to already-authorized
locations.
@@ -133,6 +137,8 @@ classification:
- `unknown` covers incomplete captures, layers outside runtime scope, and
unresolved correspondence. Nodes outside a mapped artifact prefix stay
unknown even when another prefix in the same static layer was observed.
+ Nodes beneath a mapped artifact prefix also stay unknown when a runtime
+ script or worker under that mapping has no artifact path.
None of these states proves feature execution, initialization order,
reachability, or causality. In particular, `not-observed-in-capture` never means
diff --git a/src/domain/javascript/javascriptRuntimeLoadState.ts b/src/domain/javascript/javascriptRuntimeLoadState.ts
index a1d064efc..87ba847e8 100644
--- a/src/domain/javascript/javascriptRuntimeLoadState.ts
+++ b/src/domain/javascript/javascriptRuntimeLoadState.ts
@@ -9,7 +9,7 @@ import type { RuntimeReconciliationEntity } from "./javascriptRuntimeReconciliat
import {
createStaticRuntimeScope,
mappedRuntimePaths,
- staticNodeWithinRuntimeScope,
+ staticNodeRuntimeScope,
} from "./javascriptRuntimeStaticCandidates.js";
export interface StaticLoadStateProjection {
@@ -93,8 +93,8 @@ const classifyLayer = (
? "static-or-runtime-coverage-incomplete"
: "ambiguous-runtime-candidate",
});
- const nodeWithinScope = staticNodeWithinRuntimeScope(node, runtimeScope);
- if (layerComplete && nodeWithinScope)
+ const membership = staticNodeRuntimeScope(node, runtimeScope);
+ if (layerComplete && membership === "within")
return state(layer, node, {
status: "not-observed-in-capture",
runtimeNodeIds: [],
@@ -103,9 +103,10 @@ const classifyLayer = (
return state(layer, node, {
status: "unknown",
runtimeNodeIds: [],
- reason: !nodeWithinScope
- ? "layer-outside-runtime-scope"
- : "static-or-runtime-coverage-incomplete",
+ reason:
+ membership === "outside"
+ ? "layer-outside-runtime-scope"
+ : "static-or-runtime-coverage-incomplete",
});
});
return { states };
diff --git a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts
index 5c3ef04f6..a870a8fe7 100644
--- a/src/domain/javascript/javascriptRuntimeStaticCandidates.ts
+++ b/src/domain/javascript/javascriptRuntimeStaticCandidates.ts
@@ -42,9 +42,14 @@ export interface MappedRuntimePath {
/** Precomputed mapping scope and graph paths for static load-state decisions. */
export interface StaticRuntimeScope {
readonly artifactPrefixes: readonly string[];
+ /** Prefixes beneath which a runtime script or worker had no artifact path. */
+ readonly untranslatedArtifactPrefixes: readonly string[];
readonly pathsByNodeId: ReadonlyMap<string, readonly string[]>;
}
+/** Where one static node falls relative to the runtime-exercised mappings. */
+export type StaticRuntimeScopeMembership = "outside" | "within" | "uncertain";
+
/** Index statically reconstructed entities that can receive runtime matches. */
export const collectStaticRuntimeCandidates = (
layers: readonly ParsedStaticLayer[],
@@ -79,13 +84,9 @@ export const mappedRuntimePaths = (
});
}
for (const mapping of layer.runtimeMappings) {
- const suffix =
- mapping.kind === "file-root" && entity.location.kind === "file"
- ? pathBelowRoot(mapping.root, entity.location.value)
- : mapping.kind === "url-prefix" && entity.location.kind === "url"
- ? pathBelowUrlPrefix(mapping.prefix, entity.location.value)
- : null;
- if (suffix === null) continue;
+ const translation = translateRuntimeLocation(mapping, entity.location);
+ if (translation.kind !== "translated") continue;
+ const suffix = translation.suffix;
const path = safeArtifactPath(mapping.artifact_prefix, suffix);
if (path === null) continue;
paths.push({
@@ -107,18 +108,27 @@ export const createStaticRuntimeScope = (
layer: ParsedStaticLayer,
entities: readonly RuntimeReconciliationEntity[],
): StaticRuntimeScope => ({
- artifactPrefixes: activeArtifactPrefixes(layer, entities),
+ ...exercisedArtifactPrefixes(layer, entities),
pathsByNodeId: indexStaticPaths(layer.graph),
});
-/** Whether one static node falls under a mapping exercised by this runtime set. */
-export const staticNodeWithinRuntimeScope = (
+/**
+ * Classify one static node against the mappings exercised by this runtime set.
+ * A node beneath a prefix whose runtime script or worker location could not be
+ * translated is uncertain: that location may name it, so absence is unproven.
+ */
+export const staticNodeRuntimeScope = (
node: ApplicationNode,
scope: StaticRuntimeScope,
-): boolean =>
- (scope.pathsByNodeId.get(node.node_id) ?? []).some((path) =>
- scope.artifactPrefixes.some((prefix) => pathWithinPrefix(path, prefix)),
- );
+): StaticRuntimeScopeMembership => {
+ const paths = scope.pathsByNodeId.get(node.node_id) ?? [];
+ const beneath = (prefixes: readonly string[]): boolean =>
+ paths.some((path) =>
+ prefixes.some((prefix) => pathWithinPrefix(path, prefix)),
+ );
+ if (beneath(scope.untranslatedArtifactPrefixes)) return "uncertain";
+ return beneath(scope.artifactPrefixes) ? "within" : "outside";
+};
const candidatesForLayer = (
layer: ParsedStaticLayer,
@@ -391,38 +401,87 @@ const absoluteFilePathStyle = (path: string): "posix" | "windows" | null => {
return path.startsWith("/") ? "posix" : null;
};
-const pathBelowUrlPrefix = (prefix: string, value: string): string | null => {
- let base: URL;
- let candidate: URL;
- try {
- base = new URL(prefix);
- candidate = new URL(value);
- } catch (cause: unknown) {
- // Invalid input is represented by the null return.
- void cause;
- return null;
- }
+/**
+ * How a runtime location relates to one mapping: outside it, beneath it with
+ * an artifact-relative suffix, or beneath it without any artifact path.
+ */
+type RuntimeLocationTranslation =
+ | { readonly kind: "outside" }
+ | { readonly kind: "untranslatable" }
+ | { readonly kind: "translated"; readonly suffix: string };
+
+const OUTSIDE_MAPPING: RuntimeLocationTranslation = { kind: "outside" };
+
+const translateRuntimeLocation = (
+ mapping: ParsedStaticLayer["runtimeMappings"][number],
+ location: RuntimeReconciliationEntity["location"],
+): RuntimeLocationTranslation => {
+ if (mapping.kind === "url-prefix")
+ return location.kind === "url"
+ ? pathBelowUrlPrefix(mapping.prefix, location.value)
+ : OUTSIDE_MAPPING;
+ const suffix =
+ location.kind === "file"
+ ? pathBelowRoot(mapping.root, location.value)
+ : null;
+ return suffix === null ? OUTSIDE_MAPPING : { kind: "translated", suffix };
+};
+
+const pathBelowUrlPrefix = (
+ prefix: string,
+ value: string,
+): RuntimeLocationTranslation => {
+ const base = URL.parse(prefix);
+ const candidate = URL.parse(value);
if (
+ base === null ||
+ candidate === null ||
base.origin !== candidate.origin ||
!candidate.pathname.startsWith(base.pathname)
)
- return null;
- try {
- return relativeLogicalPath(
- decodeURIComponent(candidate.pathname.slice(base.pathname.length)),
- );
- } catch (cause: unknown) {
- // Undecodable path segments are represented by the null return.
- void cause;
- return null;
+ return OUTSIDE_MAPPING;
+ const encoded = candidate.pathname.slice(base.pathname.length);
+ // The prefix itself names the mapped directory or a page, not a file below it.
+ if (encoded === "") return OUTSIDE_MAPPING;
+ const decoded = decodeUrlPathSegments(encoded);
+ const suffix = decoded === null ? null : relativeLogicalPath(decoded);
+ return suffix === null
+ ? { kind: "untranslatable" }
+ : { kind: "translated", suffix };
+};
+
+/**
+ * Decode each URL path segment separately. An encoded slash is a byte inside
+ * one segment, never an artifact directory boundary, so a segment containing
+ * one after decoding has no artifact path. An encoded backslash stays a literal
+ * name byte, and malformed or non-UTF-8 escapes have no artifact path.
+ */
+const decodeUrlPathSegments = (value: string): string | null => {
+ const segments: string[] = [];
+ for (const segment of value.split("/")) {
+ let decoded: string;
+ try {
+ decoded = decodeURIComponent(segment);
+ } catch (cause: unknown) {
+ // Malformed escapes are represented by the null return.
+ void cause;
+ return null;
+ }
+ if (decoded.includes("/")) return null;
+ segments.push(decoded);
}
+ return segments.join("/");
};
-const activeArtifactPrefixes = (
+const exercisedArtifactPrefixes = (
layer: ParsedStaticLayer,
entities: readonly RuntimeReconciliationEntity[],
-): string[] => {
+): Pick<
+ StaticRuntimeScope,
+ "artifactPrefixes" | "untranslatedArtifactPrefixes"
+> => {
const prefixes = new Set<string>();
+ const untranslated = new Set<string>();
if (
layer.result.format === "directory" &&
entities.some(
@@ -432,18 +491,26 @@ const activeArtifactPrefixes = (
)
)
prefixes.add("");
- for (const mapping of layer.runtimeMappings)
- if (
- entities.some(({ location }) =>
- mapping.kind === "file-root" && location.kind === "file"
- ? pathBelowRoot(mapping.root, location.value) !== null
- : mapping.kind === "url-prefix" && location.kind === "url"
- ? pathBelowUrlPrefix(mapping.prefix, location.value) !== null
- : false,
+ for (const mapping of layer.runtimeMappings) {
+ const prefix = normalizeArtifactPrefix(mapping.artifact_prefix);
+ for (const entity of entities) {
+ if (prefixes.has(prefix) && untranslated.has(prefix)) break;
+ const translation = translateRuntimeLocation(mapping, entity.location);
+ if (translation.kind === "translated") prefixes.add(prefix);
+ // Only scripts and workers can correspond to load-state nodes.
+ else if (
+ translation.kind === "untranslatable" &&
+ (entity.kind === "script" || entity.kind === "worker")
)
- )
- prefixes.add(normalizeArtifactPrefix(mapping.artifact_prefix));
- return [...prefixes].sort(compareUnicodeCodePoints);
+ untranslated.add(prefix);
+ }
+ }
+ return {
+ artifactPrefixes: [...prefixes].sort(compareUnicodeCodePoints),
+ untranslatedArtifactPrefixes: [...untranslated].sort(
+ compareUnicodeCodePoints,
+ ),
+ };
};
const normalizeArtifactPrefix = (prefix: string): string =>
diff --git a/tests/boundary/cli/runtimeUrlPrefixMappingCli.test.ts b/tests/boundary/cli/runtimeUrlPrefixMappingCli.test.ts
new file mode 100644
index 000000000..c09e6ceda
--- /dev/null
+++ b/tests/boundary/cli/runtimeUrlPrefixMappingCli.test.ts
@@ -0,0 +1,279 @@
+import { execFile } from "node:child_process";
+import { mkdir, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { promisify } from "node:util";
+
+import { afterEach, expect, it } from "vitest";
+import { z } from "zod";
+
+import { createBrowserEvidence } from "../../../src/application/BrowserEvidence.js";
+import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
+import { analyzeFixture } from "../../fixtures/javascriptRuntimeReconciliation.js";
+import { passiveNetworkCapture } from "../../fixtures/passiveNetworkCapture.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+const execute = promisify(execFile);
+const INTEGRATION_TEST_TIMEOUT_MS = 30_000;
+const temporary: string[] = [];
+
+afterEach(async () => {
+ await Promise.all(
+ temporary
+ .splice(0)
+ .map(async (path) => rm(path, { recursive: true, force: true })),
+ );
+});
+
+const reconciledSchema = z.object({
+ normalized_result: z.object({
+ reconciliations: z.array(
+ z.object({
+ entity_kind: z.string(),
+ runtime_node_id: z.string(),
+ static_node_id: z.string().nullable(),
+ status: z.string(),
+ basis: z.string(),
+ confidence: z.string(),
+ reason: z.string(),
+ }),
+ ),
+ static_load_states: z.array(
+ z.object({
+ static_node_id: z.string(),
+ kind: z.string(),
+ status: z.string(),
+ reason: z.string(),
+ }),
+ ),
+ graph: z.object({
+ nodes: z.array(
+ z.object({
+ node_id: z.string(),
+ observations: z.array(
+ z.object({
+ properties: z.record(z.string(), z.unknown()),
+ }),
+ ),
+ }),
+ ),
+ edges: z.array(
+ z.object({
+ relation: z.string(),
+ source_node_id: z.string(),
+ target_node_id: z.string(),
+ }),
+ ),
+ }),
+ }),
+});
+
+/** Reconcile one source-free browser capture under `${origin}/assets/`. */
+const reconcileThroughCli = async (
+ root: string,
+ analysis: Awaited<ReturnType<typeof analyzeFixture>>,
+ suffixes: readonly string[],
+) => {
+ const inspection = passiveNetworkCapture([]);
+ const origin = inspection.target.origin;
+ inspection.frames = [
+ { frame_id: "frame-main", parent_frame_id: null, url: origin, origin },
+ ];
+ inspection.scripts = {
+ total: suffixes.length,
+ items: suffixes.map((suffix, index) => ({
+ script_key: `scr_${String(index).padStart(64, "0")}`,
+ frame_id: "frame-main",
+ url: `${origin}/assets/${suffix}`,
+ origin,
+ cdp_hash: null,
+ length: null,
+ is_module: true,
+ language: "JavaScript",
+ source_map_url: null,
+ resource_reconciliation: {
+ status: "unmatched",
+ reason: "no_exact_sanitized_url",
+ },
+ source: { included: false, reason: "not-requested" },
+ })),
+ };
+ const runtime = createBrowserEvidence(
+ "inspect_web_page",
+ inspectWebPageInputSchema.parse({
+ cdp_endpoint: "http://127.0.0.1:9222",
+ target_id: "page",
+ allowed_origins: [origin],
+ observation_ms: 0,
+ include_script_sources: false,
+ }),
+ inspection,
+ {
+ id: "rea-cdp-browser",
+ name: "REA Chrome DevTools Protocol observation provider",
+ version: "2",
+ },
+ );
+ const inputPath = join(root, "reconciliation.json");
+ await writeFile(
+ inputPath,
+ JSON.stringify({
+ static_layers: [
+ {
+ role: "application",
+ analysis,
+ runtime_mappings: [
+ {
+ kind: "url-prefix",
+ prefix: `${origin}/assets/`,
+ artifact_prefix: "",
+ },
+ ],
+ },
+ ],
+ runtime_observations: [runtime],
+ }),
+ );
+ const { stdout, stderr } = await execute(
+ process.execPath,
+ ["scripts/rea.mjs", "reconcile-javascript-runtime", inputPath, "--json"],
+ { cwd: process.cwd(), maxBuffer: 16 * 1_024 * 1_024 },
+ );
+ expect(stderr).toBe("");
+ const result = reconciledSchema.parse(JSON.parse(stdout)).normalized_result;
+ const nodes = new Map(result.graph.nodes.map((node) => [node.node_id, node]));
+ const property = (nodeId: string | null, key: string) =>
+ nodeId === null
+ ? null
+ : (nodes
+ .get(nodeId)
+ ?.observations.map(({ properties }) => properties[key])
+ .find((value) => typeof value === "string") ?? null);
+ const pathOf = (nodeId: string | null) => property(nodeId, "path");
+ const scripts = new Map(
+ result.reconciliations
+ .filter(({ entity_kind }) => entity_kind === "script")
+ .map((item) => [
+ String(property(item.runtime_node_id, "location")).slice(
+ `${origin}/assets/`.length,
+ ),
+ {
+ status: item.status,
+ basis: item.basis,
+ reason: item.reason,
+ path: pathOf(item.static_node_id),
+ observed_as: result.graph.edges.filter(
+ ({ relation, source_node_id, target_node_id }) =>
+ relation === "observed_as" &&
+ [source_node_id, target_node_id].includes(item.runtime_node_id),
+ ).length,
+ },
+ ]),
+ );
+ const loadStates = new Map(
+ result.static_load_states
+ .filter(({ kind }) => kind === "javascript-asset")
+ .map((state) => [
+ pathOf(state.static_node_id),
+ { status: state.status, reason: state.reason },
+ ]),
+ );
+ return { scripts, loadStates };
+};
+
+const unmappedLocation = {
+ status: "unknown",
+ basis: "none",
+ reason: "no-authorized-location-mapping",
+ path: null,
+ observed_as: 0,
+};
+
+const mappedLocation = (path: string) => ({
+ status: "matched",
+ basis: "operator-url-mapping",
+ reason: "unique-location-match",
+ path,
+ observed_as: 1,
+});
+
+it(
+ "keeps an encoded slash inside one URL segment rather than naming a nested artifact",
+ async () => {
+ const root = await createTestTempDirectory("rea-url-prefix-segments-");
+ temporary.push(root);
+ await mkdir(join(root, "child"));
+ await Promise.all([
+ writeFile(join(root, "child", "part.js"), "export const nested = 1;\n"),
+ writeFile(join(root, "child%2Fpart.js"), "export const literal = 2;\n"),
+ writeFile(join(root, "sp ace.js"), "export const space = 3;\n"),
+ writeFile(join(root, "é.js"), "export const accent = 4;\n"),
+ ]);
+ const analysis = await analyzeFixture(root);
+
+ const { scripts } = await reconcileThroughCli(root, analysis, [
+ "child/part.js",
+ "child/part.js?v=%2F#f%2F",
+ "child%2Fpart.js",
+ "child%2fpart.js",
+ "child%252Fpart.js",
+ "sp%20ace.js",
+ "%C3%A9.js",
+ "child/%E0.js",
+ ]);
+
+ expect(Object.fromEntries(scripts)).toEqual({
+ "child/part.js": mappedLocation("child/part.js"),
+ "child/part.js?v=%2F#f%2F": mappedLocation("child/part.js"),
+ "child%2Fpart.js": unmappedLocation,
+ "child%2fpart.js": unmappedLocation,
+ "child%252Fpart.js": mappedLocation("child%2Fpart.js"),
+ "sp%20ace.js": mappedLocation("sp ace.js"),
+ "%C3%A9.js": mappedLocation("é.js"),
+ "child/%E0.js": unmappedLocation,
+ });
+ },
+ INTEGRATION_TEST_TIMEOUT_MS,
+);
+
+it(
+ "keeps static load state unknown beneath a prefix whose runtime script had no artifact path",
+ async () => {
+ const root = await createTestTempDirectory("rea-url-prefix-load-state-");
+ temporary.push(root);
+ await mkdir(join(root, "child"));
+ await Promise.all([
+ writeFile(join(root, "child", "part.js"), "export const nested = 1;\n"),
+ writeFile(join(root, "other.js"), "export const other = 2;\n"),
+ ]);
+ const analysis = await analyzeFixture(root);
+
+ const control = await reconcileThroughCli(root, analysis, ["other.js"]);
+ expect(Object.fromEntries(control.loadStates)).toEqual({
+ "child/part.js": {
+ status: "not-observed-in-capture",
+ reason: "not-observed-in-bounded-capture",
+ },
+ "other.js": {
+ status: "loaded",
+ reason: "runtime-script-correspondence",
+ },
+ });
+
+ const encoded = await reconcileThroughCli(root, analysis, [
+ "other.js",
+ "child%2Fpart.js",
+ ]);
+ expect(encoded.scripts.get("child%2Fpart.js")).toEqual(unmappedLocation);
+ expect(Object.fromEntries(encoded.loadStates)).toEqual({
+ "child/part.js": {
+ status: "unknown",
+ reason: "static-or-runtime-coverage-incomplete",
+ },
+ "other.js": {
+ status: "loaded",
+ reason: "runtime-script-correspondence",
+ },
+ });
+ },
+ INTEGRATION_TEST_TIMEOUT_MS,
+);