Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(javascript): reconstruct HTML script loads accurately (#1042)
Rudy Mizrahi Celekli1154f2f03c0b
diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.mdindex 52038d16d..47eca1e8c 100644--- a/docs/javascript-artifact-reconstruction.md+++ b/docs/javascript-artifact-reconstruction.md@@ -230,3 +230,9 @@ such as `./web3:app.js` still names a local artifact.Package `exports` fallback arrays are supported both at the top level and underthe root `"."` entry. The resolver selects targets in declared order using thesame conditional and invalid-entry handling as nested exports arrays.++HTML script references resolve to exact inventoried files after applying the+document base and query/fragment rules. CommonJS module lookups retain extension+and directory resolution.+Unresolved HTML references retain their declaration, source range, and resolution+reason in the renderer observations.diff --git a/package-lock.json b/package-lock.jsonindex e47340616..1bb44c60c 100644--- a/package-lock.json+++ b/package-lock.json@@ -31,6 +31,7 @@"isomorphic-git": "1.38.7","jsonc-parser": "3.3.1","lossless-json": "4.3.1",+ "parse5": "8.0.1","pino": "10.4.0","playwright-core": "1.58.2","plist": "5.0.0",@@ -6339,6 +6340,30 @@"integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==","license": "(MIT AND Zlib)"},+ "node_modules/parse5": {+ "version": "8.0.1",+ "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz",+ "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==",+ "license": "MIT",+ "dependencies": {+ "entities": "^8.0.0"+ },+ "funding": {+ "url": "https://github.com/inikulin/parse5?sponsor=1"+ }+ },+ "node_modules/parse5/node_modules/entities": {+ "version": "8.1.0",+ "resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz",+ "integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==",+ "license": "BSD-2-Clause",+ "engines": {+ "node": ">=20.19.0"+ },+ "funding": {+ "url": "https://github.com/fb55/entities?sponsor=1"+ }+ },"node_modules/path-key": {"version": "3.1.1","resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",diff --git a/package.json b/package.jsonindex 8643ff966..e47a8fbed 100644--- a/package.json+++ b/package.json@@ -225,6 +225,7 @@"isomorphic-git": "1.38.7","jsonc-parser": "3.3.1","lossless-json": "4.3.1",+ "parse5": "8.0.1","pino": "10.4.0","playwright-core": "1.58.2","plist": "5.0.0",diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.tsindex 9422eacff..b390854e0 100644--- a/src/application/javascript/JavaScriptArtifactAnalysis.ts+++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts@@ -25,6 +25,7 @@ import type {JavaScriptSourceMapOriginal,} from "./JavaScriptArtifactAnalysisTypes.js";import { analyzeJavaScriptJsonModule } from "./JavaScriptJsonModules.js";+import { htmlArtifactReferences } from "../../domain/javascript/htmlArtifactReferences.js";interface MutableArtifactAnalysis {readonly files: AnalyzedJavaScriptArtifactFile[];@@ -205,26 +206,13 @@ const parseHtmlScripts = (path: string,text: string,): JavaScriptHtmlScriptObservation[] => {- const scripts: JavaScriptHtmlScriptObservation[] = [];- const baseHref = htmlBaseHref(text);- const pattern = /<script\b[^>]*\bsrc\s*=\s*(["'])([^"']+)\1[^>]*>/giu;- for (const match of text.matchAll(pattern)) {- const script = match[2];- if (script === undefined) continue;- const start = match.index;- scripts.push({- html_path: path,- script_path: script,- base_href: baseHref,- location: rangeForOffsets(text, start, start + match[0].length),- });- }- return scripts;-};--const htmlBaseHref = (text: string): string | null => {- const match = /<base\b[^>]*\bhref\s*=\s*(["'])([^"']+)\1[^>]*>/iu.exec(text);- return match?.[2] ?? null;+ const { scripts, baseHref } = htmlArtifactReferences(text);+ return scripts.map(({ scriptPath, startOffset, endOffset }) => ({+ html_path: path,+ script_path: scriptPath,+ base_href: baseHref,+ location: rangeForOffsets(text, startOffset, endOffset),+ }));};const parseSourceMap = (diff --git a/src/application/javascript/JavaScriptArtifactGraphDocuments.ts b/src/application/javascript/JavaScriptArtifactGraphDocuments.tsindex 18258ef13..f17293f5e 100644--- a/src/application/javascript/JavaScriptArtifactGraphDocuments.ts+++ b/src/application/javascript/JavaScriptArtifactGraphDocuments.ts@@ -5,7 +5,10 @@ import {linkElectronRoleToAsset,type JavaScriptArtifactGraphContext,} from "./JavaScriptArtifactGraphContext.js";-import { artifactObservationEvidence } from "./JavaScriptArtifactGraphEvidence.js";+import {+ artifactObservationEvidence,+ staticInferenceEvidence,+} from "./JavaScriptArtifactGraphEvidence.js";import { completeApplicationCoverage } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js";import { resolveArtifactPathByContext } from "./JavaScriptArtifactPathResolution.js";@@ -57,7 +60,32 @@ export const addJavaScriptHtmlRoles = (htmlBaseHref: script.base_href,});const resolvedPath = resolution.resolved_path;- if (resolvedPath === null) continue;+ if (resolvedPath === null) {+ context.accumulator.addNode({+ kind: role.kind,+ identity: role.identity,+ observations: [+ {+ label: script.script_path,+ properties: {+ mechanism: "html-script-reference",+ script_path: script.script_path,+ base_href: script.base_href,+ ...resolution,+ },+ evidence: staticInferenceEvidence({+ sha256: html.sha256,+ path: html.path,+ range: script.location,+ operation: "resolve-html-script-reference",+ coverage: completeApplicationCoverage(),+ limitations: resolution.limitations,+ }),+ },+ ],+ });+ continue;+ }const resolved = context.fileNodes.get(resolvedPath);if (resolved === undefined) continue;addStaticInferenceEdge(context, {diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts b/src/application/javascript/JavaScriptArtifactPathResolution.test.tsindex 0653ac8f7..c647dfe48 100644--- a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts+++ b/src/application/javascript/JavaScriptArtifactPathResolution.test.ts@@ -676,3 +676,52 @@ const file = (? { included: false, reason: "invalid-utf8" }: { included: true, value: text },});++describe("exact HTML resource resolution", () => {+ it.each([+ ["extension", ["app.js"]],+ ["directory index", ["app/index.js"]],+ ["directory package", ["app/package.json", "app/entry.js"]],+ ] as const)("does not infer a file from %s fallback", (_name, paths) => {+ const files = fileMap([+ file("index.html", "root"),+ ...paths.map((path) =>+ file(+ path,+ "root",+ path.endsWith("package.json") ? '{"main":"entry.js"}' : "",+ ),+ ),+ ]);+ expect(+ resolve({+ declaredPath: "./app",+ sourcePath: "index.html",+ context: "html-reference",+ files,+ }),+ ).toMatchObject({ resolution_status: "not-found", resolved_path: null });+ });++ it("prefers an exact extensionless resource and retains URL suffix handling", () => {+ const files = fileMap([+ file("renderer/index.html", "root"),+ file("assets/app", "root"),+ file("assets/app.js", "root"),+ file("assets/app/index.js", "root"),+ ]);+ expect(+ resolve({+ declaredPath: "./app?cache=1#v2",+ sourcePath: "renderer/index.html",+ context: "html-reference",+ htmlBaseHref: "/assets/",+ files,+ }),+ ).toMatchObject({+ declared_path: "./app?cache=1#v2",+ resolution_status: "resolved",+ resolved_path: "assets/app",+ });+ });+});diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.ts b/src/application/javascript/JavaScriptArtifactPathResolution.tsindex 956da69e5..53b871efd 100644--- a/src/application/javascript/JavaScriptArtifactPathResolution.ts+++ b/src/application/javascript/JavaScriptArtifactPathResolution.ts@@ -93,6 +93,17 @@ const resolvePackagePath = (if (typeof candidate !== "string") return candidate;const confined = confineCandidate(input, candidate);if (typeof confined !== "string") return confined;+ if (input.context === "html-reference")+ return outcome(+ input,+ resolveFileCandidates(input, [confined]) ?? {+ resolvedPath: null,+ status: "not-found",+ limitations: [+ `The exact HTML resource ${confined} was not found among the selected application files.`,+ ],+ },+ );const resolved =mode === "files-and-index"? (resolveFileCandidates(input, [diff --git a/src/domain/javascript/htmlArtifactReferences.test.ts b/src/domain/javascript/htmlArtifactReferences.test.tsnew file mode 100644index 000000000..d5e84d5db--- /dev/null+++ b/src/domain/javascript/htmlArtifactReferences.test.ts@@ -0,0 +1,70 @@+import { expect, it } from "vitest";++import { htmlArtifactReferences } from "./htmlArtifactReferences.js";++it.each([+ ['<script data-src="decoy.js"></script>', []],+ ['<script x:src="decoy.js"></script>', []],+ [`<script title='src="decoy.js"'></script>`, []],+ ["<script src=actual.js></script>", ["actual.js"]],+ ['<script title=">" src="actual.js"></script>', ["actual.js"]],+ ['<SCRIPT SRC="actual.js"></SCRIPT>', ["actual.js"]],+ ["<script src='actual.js'></script>", ["actual.js"]],+ ['<script data-src="decoy.js" src="actual.js"></script>', ["actual.js"]],+ ['<script src="actual.js" src="decoy.js"></script>', ["actual.js"]],+ ['<script src="" src="decoy.js"></script>', []],+ ['<script src="actual.js"></script>', ["actual.js"]],+ ['<!-- <script src="decoy.js"></script> -->', []],+ ['<textarea><script src="decoy.js"></script></textarea>', []],+ ['<style><script src="decoy.js"></script></style>', []],+ [`<script>const text = '<script src="decoy.js">';</script>`, []],+ [+ '<template><base href="decoy/"><script src="decoy.js"></script></template>',+ [],+ ],+ ['<svg><script src="decoy.js"></script></svg>', []],+ [+ '<svg><foreignObject><script src="actual.js"></script></foreignObject></svg>',+ ["actual.js"],+ ],+ // The existing extraction contract does not classify script MIME types.+ ['<script type="application/json" src="actual.js"></script>', ["actual.js"]],+])("extracts document references from %s", (text, expected) => {+ expect(+ htmlArtifactReferences(text).scripts.map(({ scriptPath }) => scriptPath),+ ).toEqual(expected);+});++it("retains exact opening-tag offsets rather than the script body", () => {+ const opening = '<script\n title=">" src=actual.js>';+ const prefix = "<!doctype html>\n\n";+ const text = `${prefix}${opening}ignored body</script>`;+ expect(htmlArtifactReferences(text)).toEqual({+ baseHref: null,+ scripts: [+ {+ scriptPath: "actual.js",+ startOffset: prefix.length,+ endOffset: prefix.length + opening.length,+ },+ ],+ });+});++it.each([+ ['<base data-href="decoy/"><base href=actual/>', "actual/"],+ ['<base href="actual/" href="decoy/"><base href="later/">', "actual/"],+ ['<base href=""><base href="decoy/">', ""],+ ['<template><base href="decoy/"></template><base href="actual/">', "actual/"],+ ['<!-- <base href="decoy/"> --><base title=">" href="actual/">', "actual/"],+])("selects the first real base href from %s", (text, expected) => {+ expect(htmlArtifactReferences(text).baseHref).toBe(expected);+});++it("traverses deep valid document content without recursive stack growth", () => {+ const depth = 12_000;+ const text = `${"<div>".repeat(depth)}<script src=actual.js></script>${"</div>".repeat(depth)}`;+ expect(+ htmlArtifactReferences(text).scripts.map(({ scriptPath }) => scriptPath),+ ).toEqual(["actual.js"]);+});diff --git a/src/domain/javascript/htmlArtifactReferences.ts b/src/domain/javascript/htmlArtifactReferences.tsnew file mode 100644index 000000000..fa9945e36--- /dev/null+++ b/src/domain/javascript/htmlArtifactReferences.ts@@ -0,0 +1,55 @@+import {+ defaultTreeAdapter,+ parse,+ type DefaultTreeAdapterTypes,+} from "parse5";++interface HtmlScriptReference {+ readonly scriptPath: string;+ readonly startOffset: number;+ readonly endOffset: number;+}++/** Extract inert HTML script references and the first document base href. */+export const htmlArtifactReferences = (+ text: string,+): {+ readonly baseHref: string | null;+ readonly scripts: readonly HtmlScriptReference[];+} => {+ const pending: DefaultTreeAdapterTypes.Node[] = [+ parse(text, { sourceCodeLocationInfo: true }),+ ];+ const scripts: HtmlScriptReference[] = [];+ let baseHref: string | null = null;+ while (pending.length > 0) {+ const node = pending.pop();+ if (node === undefined) continue;+ if (+ defaultTreeAdapter.isElementNode(node) &&+ node.namespaceURI === "http://www.w3.org/1999/xhtml"+ ) {+ if (node.tagName === "base" && baseHref === null) {+ const href = node.attrs.find(({ name }) => name === "href");+ if (href !== undefined) baseHref = href.value;+ }+ if (node.tagName === "script") {+ const src = node.attrs.find(({ name }) => name === "src");+ const location = node.sourceCodeLocation?.startTag;+ if (src !== undefined && src.value !== "" && location !== undefined) {+ scripts.push({+ scriptPath: src.value,+ startOffset: location.startOffset,+ endOffset: location.endOffset,+ });+ }+ }+ }+ // Template content belongs to a separate inert document fragment. Only+ // traverse the actual document children, without recursion on deep input.+ if ("childNodes" in node) {+ for (const child of node.childNodes.toReversed()) pending.push(child);+ }+ }+ return { baseHref, scripts };+};diff --git a/tests/boundary/filesystem/javascriptHtmlEntrypoints.test.ts b/tests/boundary/filesystem/javascriptHtmlEntrypoints.test.tsnew file mode 100644index 000000000..f25d7fb9e--- /dev/null+++ b/tests/boundary/filesystem/javascriptHtmlEntrypoints.test.ts@@ -0,0 +1,87 @@+import { createHash } from "node:crypto";+import { mkdir, writeFile } from "node:fs/promises";+import { join } from "node:path";++import { expect, it } from "vitest";++import { reconstructJavaScriptArtifact } from "../../../src/application/javascript/JavaScriptArtifactReconstruction.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";++const cases = [+ ['<script data-src="decoy.js"></script>', []],+ ['<script x:src="decoy.js"></script>', []],+ [`<script title='src="decoy.js"'></script>`, []],+ ["<script src=actual.js></script>", ["actual.js"]],+ ['<script title=">" src="actual.js"></script>', ["actual.js"]],+ ['<script src="actual.js"></script>', ["actual.js"]],+ ["<script src='actual.js'></script>", ["actual.js"]],+ ['<script data-src="decoy.js" src="actual.js"></script>', ["actual.js"]],+ ['<!-- <script src="decoy.js"></script> -->', []],+ ['<textarea><script src="decoy.js"></script></textarea>', []],+ ['<template><script src="decoy.js"></script></template>', []],+ ['<script src="actual.js" src="decoy.js"></script>', ["actual.js"]],+] as const;++it.each(cases)(+ "projects local loads from actual HTML attributes in %s",+ async (html, expected) => {+ const root = await createTestTempDirectory("rea-html-entrypoints-");+ await Promise.all([+ writeFile(join(root, "index.html"), html),+ writeFile(join(root, "actual.js"), "globalThis.ownedActual = true;"),+ writeFile(join(root, "decoy.js"), "globalThis.ownedDecoy = true;"),+ ]);+ const result = await reconstructJavaScriptArtifact({ input_path: root });+ const loads = result.graph.edges.filter(+ ({ relation, properties }) =>+ relation === "loads" && properties.script_path !== undefined,+ );+ expect(loads.map(({ properties }) => properties.resolved_path)).toEqual(+ expected,+ );+ },+);++it("preserves source digest, opening-tag range and decoded local base resolution", async () => {+ const root = await createTestTempDirectory("rea-html-entrypoint-evidence-");+ const opening = '<script title=">" src="actual.js">';+ const prefix = '<base data-href="decoy/"><base href=assets/>\n';+ const html = `${prefix}${opening}</script>`;+ await mkdir(join(root, "assets"));+ await Promise.all([+ writeFile(join(root, "index.html"), html),+ writeFile(+ join(root, "assets", "actual.js"),+ "globalThis.ownedActual = true;",+ ),+ ]);+ const result = await reconstructJavaScriptArtifact({ input_path: root });+ const loads = result.graph.edges.filter(+ ({ relation, properties }) =>+ relation === "loads" && properties.script_path !== undefined,+ );+ expect(loads).toHaveLength(1);+ expect(loads[0]).toMatchObject({+ properties: {+ script_path: "actual.js",+ base_href: "assets/",+ resolved_path: "assets/actual.js",+ },+ evidence: {+ authority: "static-relationship-inference",+ artifact: {+ available: true,+ sha256: createHash("sha256").update(html).digest("hex"),+ },+ location: {+ available: true,+ value: {+ kind: "source-range",+ source: "index.html",+ start: { line: 2, column: 0 },+ end: { line: 2, column: opening.length },+ },+ },+ },+ });+});diff --git a/tests/boundary/filesystem/javascriptHtmlResourcePaths.test.ts b/tests/boundary/filesystem/javascriptHtmlResourcePaths.test.tsnew file mode 100644index 000000000..1edd18a86--- /dev/null+++ b/tests/boundary/filesystem/javascriptHtmlResourcePaths.test.ts@@ -0,0 +1,133 @@+import { mkdir, writeFile } from "node:fs/promises";+import { createRequire } from "node:module";+import { dirname, join, relative, sep } from "node:path";++import { expect, it } from "vitest";++import { reconstructJavaScriptArtifact } from "../../../src/application/javascript/JavaScriptArtifactReconstruction.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";++const cases: readonly {+ name: string;+ html: string;+ files: Readonly<Record<string, string>>;+ expected: readonly string[];+}[] = [+ {+ name: "extension fallback",+ html: '<script src="./app"></script>',+ files: { "app.js": "globalThis.owned = true;" },+ expected: [],+ },+ {+ name: "directory index",+ html: '<script src="./app"></script>',+ files: { "app/index.js": "globalThis.owned = true;" },+ expected: [],+ },+ {+ name: "directory package",+ html: '<script src="./app"></script>',+ files: {+ "app/package.json": '{"main":"entry.js"}',+ "app/entry.js": "globalThis.owned = true;",+ },+ expected: [],+ },+ {+ name: "base directory",+ html: '<base href="assets/"><script src="./app"></script>',+ files: { "assets/app.js": "globalThis.owned = true;" },+ expected: [],+ },+ {+ name: "exact base resource",+ html: '<base href="assets/"><script src="./app.js?cache=1#v2"></script>',+ files: {+ "assets/app.js": "globalThis.owned = true;",+ "app.js": "globalThis.decoy = true;",+ },+ expected: ["assets/app.js"],+ },+ {+ name: "exact script resource",+ html: '<script src="./app.js?cache=1#v2"></script>',+ files: { "app.js": "globalThis.owned = true;" },+ expected: ["app.js"],+ },+];++it.each(cases)(+ "uses exact HTML resource identity for $name",+ async ({ html, files, expected }) => {+ const root = await createTestTempDirectory("rea-html-resources-");+ await Promise.all(+ Object.entries({ ...files, "index.html": html }).map(+ async ([path, text]) => {+ await mkdir(dirname(join(root, path)), { recursive: true });+ await writeFile(join(root, path), text);+ },+ ),+ );+ const result = await reconstructJavaScriptArtifact({ input_path: root });+ const loads = result.graph.edges.filter(+ ({ relation, properties }) =>+ relation === "loads" && properties.script_path !== undefined,+ );+ expect(loads.map(({ properties }) => properties.resolved_path)).toEqual(+ expected,+ );+ if (expected.length === 0) {+ const reference = result.graph.nodes+ .flatMap(({ observations }) => observations)+ .find(+ ({ properties }) => properties.mechanism === "html-script-reference",+ );+ expect(reference).toMatchObject({+ properties: {+ script_path: "./app",+ resolution_context: "html-reference",+ resolution_status: "not-found",+ resolved_path: null,+ },+ evidence: {+ authority: "static-relationship-inference",+ location: {+ available: true,+ value: { kind: "source-range", source: "index.html" },+ },+ limitations: expect.arrayContaining([+ expect.stringContaining("exact HTML resource"),+ ]),+ },+ });+ }+ },+);++it("retains CommonJS extension resolution against Node", async () => {+ const root = await createTestTempDirectory("rea-html-node-control-");+ await Promise.all([+ writeFile(+ join(root, "main.cjs"),+ 'const value = require("./app"); module.exports = value;',+ ),+ writeFile(join(root, "app.js"), 'module.exports = "actual";'),+ ]);+ const expected = relative(+ root,+ createRequire(join(root, "main.cjs")).resolve("./app"),+ )+ .split(sep)+ .join("/");+ const result = await reconstructJavaScriptArtifact({ input_path: root });+ const edge = result.graph.edges.find(+ ({ properties }) =>+ properties.module_link_kind === "require" &&+ properties.specifier === "./app",+ );+ expect(edge?.properties).toMatchObject({+ resolution_status: "resolved",+ resolved_path: expected,+ });+});