Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(javascript): mark resource-skipped files parse_status skipped (#3852)

Saim Shafique77c8a1ba321c
diff --git a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts
index ef174affb..500fb57cf 100644
--- a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts
+++ b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts
@@ -170,6 +170,22 @@ export interface AnalyzedJavaScriptArtifactFile<
};
}
+/**
+ * A JavaScript source selected for analysis whose worker produced no facts.
+ * The analysis failure carries the reason; the null parse status alone reads
+ * as "nothing there" instead of "not analyzed".
+ */
+export const isSkippedJavaScriptAnalysis = (
+ entry: Pick<
+ AnalyzedJavaScriptArtifactFile,
+ "file" | "javascript" | "analysis_failure"
+ >,
+): boolean =>
+ entry.file.kind === "javascript" &&
+ entry.file.text.included &&
+ entry.javascript === null &&
+ entry.analysis_failure !== undefined;
+
/** Complete static-analysis projection before graph construction. */
export interface JavaScriptArtifactAnalysis<
SemanticIr extends JavaScriptModuleSemanticIr = JavaScriptSemanticIr,
diff --git a/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts b/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts
new file mode 100644
index 000000000..dbfc19d76
--- /dev/null
+++ b/src/application/javascript/JavaScriptArtifactGraphStructure.test.ts
@@ -0,0 +1,148 @@
+import { describe, expect, it } from "vitest";
+
+import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
+import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
+import { analyzeJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js";
+import type {
+ AnalyzedJavaScriptArtifactFile,
+ JavaScriptModuleArtifactAnalysis,
+} from "./JavaScriptArtifactAnalysisTypes.js";
+import { javascriptArtifactAnalysisScopes } from "./JavaScriptArtifactAnalysisTypes.js";
+import { JavaScriptArtifactGraphAccumulator } from "./JavaScriptArtifactGraphAccumulator.js";
+import type { JavaScriptArtifactGraphContext } from "./JavaScriptArtifactGraphContext.js";
+import {
+ addJavaScriptArtifactFiles,
+ createJavaScriptArtifactRootNode,
+} from "./JavaScriptArtifactGraphStructure.js";
+
+const SHA256 = "a".repeat(64);
+const REASON =
+ "main.js: Resource constraint during analyze_javascript_application: Owned JavaScript worker exhausted its heap while analyzing main.js during static";
+
+const skippedFile = (): JavaScriptArtifactFile => ({
+ path: "main.js",
+ container_sha256: SHA256,
+ sha256: SHA256,
+ bytes: 5000107,
+ inventory_artifact_id: `art_${SHA256}`,
+ kind: "javascript",
+ unpacked: false,
+ text: { included: true, value: "const {ipcMain}=require('electron');\n" },
+});
+
+const skippedEntry = (
+ file: JavaScriptArtifactFile,
+): AnalyzedJavaScriptArtifactFile => ({
+ file,
+ javascript: null,
+ semantic: null,
+ analysis_failure: {
+ scope: javascriptArtifactAnalysisScopes["javascript"],
+ status: "interrupted",
+ reason: REASON,
+ error: { code: "resource_constraint" },
+ limits: null,
+ },
+});
+
+const analysisFor = (
+ entries: readonly AnalyzedJavaScriptArtifactFile[],
+): JavaScriptModuleArtifactAnalysis => ({
+ files: entries,
+ packages: [],
+ json_modules: [],
+ html_scripts: [],
+ html_inline_scripts: [],
+ source_maps: [],
+ visited_ast_nodes: 0,
+ findings: 0,
+ modules: 0,
+ parse_failures: 0,
+ limitations: [],
+});
+
+const contextFor = (
+ file: JavaScriptArtifactFile,
+ analysis: JavaScriptModuleArtifactAnalysis,
+): JavaScriptArtifactGraphContext => {
+ const accumulator = new JavaScriptArtifactGraphAccumulator();
+ const snapshot = {
+ manifest: {
+ root_artifact_id: `art_${SHA256}`,
+ root_sha256: SHA256,
+ root_format: "directory",
+ manifest_id: `agm_${SHA256}`,
+ graph_sha256: SHA256,
+ },
+ nodes: [{ artifact_id: `art_${SHA256}`, sha256: SHA256, size: 1 }],
+ } as unknown as ArtifactInventorySnapshot;
+ const fileSet = {
+ files: [file],
+ unavailable_relevant_files: [],
+ containers: [],
+ text_bytes_read: file.bytes,
+ invalid_utf8_files: 0,
+ };
+ return {
+ accumulator,
+ snapshot,
+ fileSet,
+ analysis,
+ directoryPaths: new Set<string>(),
+ root: createJavaScriptArtifactRootNode(accumulator, snapshot),
+ filesByPath: new Map([[file.path, file]]),
+ unanalyzedFiles: undefined,
+ fileNodes: new Map(),
+ assetNodes: new Map(),
+ sourceModuleNodes: new Map(),
+ chunkNodes: new Map(),
+ moduleNodes: new Map(),
+ containerNodes: new Map(),
+ };
+};
+
+const fileObservation = (context: JavaScriptArtifactGraphContext) => {
+ const node = context.accumulator
+ .nodes()
+ .find(({ kind }) => kind === "javascript-asset");
+ if (node === undefined) throw new Error("Expected a javascript-asset node");
+ const observation = node.observations[0];
+ if (observation === undefined) throw new Error("Expected a file observation");
+ return observation.properties;
+};
+
+describe("skipped JavaScript parse reporting", () => {
+ it("marks parse_status skipped for a JavaScript file whose worker analysis failed", () => {
+ const file = skippedFile();
+ const context = contextFor(file, analysisFor([skippedEntry(file)]));
+ addJavaScriptArtifactFiles(context);
+ expect(fileObservation(context).parse_status).toBe("skipped");
+ });
+
+ it("keeps parse_status null for a JavaScript file never attempted", () => {
+ const file = skippedFile();
+ const entry: AnalyzedJavaScriptArtifactFile = {
+ file,
+ javascript: null,
+ semantic: null,
+ };
+ const context = contextFor(file, analysisFor([entry]));
+ addJavaScriptArtifactFiles(context);
+ expect(fileObservation(context).parse_status).toBeNull();
+ });
+
+ it("keeps the parsed status for an analyzed JavaScript file", () => {
+ const file = skippedFile();
+ const javascript = analyzeJavaScriptStaticSource(
+ "const {ipcMain}=require('electron');\n",
+ );
+ const entry: AnalyzedJavaScriptArtifactFile = {
+ file,
+ javascript,
+ semantic: null,
+ };
+ const context = contextFor(file, analysisFor([entry]));
+ addJavaScriptArtifactFiles(context);
+ expect(fileObservation(context).parse_status).toBe(javascript.parse_status);
+ });
+});
diff --git a/src/application/javascript/JavaScriptArtifactGraphStructure.ts b/src/application/javascript/JavaScriptArtifactGraphStructure.ts
index 43c00d93b..6ec343af1 100644
--- a/src/application/javascript/JavaScriptArtifactGraphStructure.ts
+++ b/src/application/javascript/JavaScriptArtifactGraphStructure.ts
@@ -15,6 +15,7 @@ import type {
JavaScriptArtifactFile,
} from "../../domain/javascript/javascriptArtifactFiles.js";
import type { JavaScriptArtifactGraphAccumulator } from "./JavaScriptArtifactGraphAccumulator.js";
+import { isSkippedJavaScriptAnalysis } from "./JavaScriptArtifactAnalysisTypes.js";
import {
addArtifactContainsEdge,
addUnavailableStaticParseScope,
@@ -145,6 +146,7 @@ export const addJavaScriptArtifactFiles = (
file,
analyzed.javascript,
jsonValue,
+ isSkippedJavaScriptAnalysis(analyzed),
);
if (
analyzed.application_projection_failure !== undefined &&
@@ -553,6 +555,7 @@ const createFileTarget = (
file: JavaScriptArtifactFile,
javascript: JavaScriptModuleArtifactAnalysis["files"][number]["javascript"],
json: JavaScriptModuleArtifactAnalysis["json_modules"][number] | undefined,
+ skipped: boolean,
): ApplicationNode => {
const kind = artifactFileNodeKind(file.kind);
return context.accumulator.addNode({
@@ -572,7 +575,8 @@ const createFileTarget = (
unpacked: file.unpacked,
file_kind: file.kind,
text_status: file.text.included ? "included" : file.text.reason,
- parse_status: javascript?.parse_status ?? null,
+ parse_status:
+ javascript?.parse_status ?? (skipped ? "skipped" : null),
json_parse_status: json?.status ?? null,
json_top_level_keys: json?.top_level_keys ?? [],
omitted_json_top_level_keys: json?.omitted_top_level_keys ?? 0,
diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts
index cb7d66089..5f40f2928 100644
--- a/src/application/javascript/JavaScriptArtifactReconstruction.ts
+++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts
@@ -20,7 +20,10 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySn
import type { ArtifactInventoryPartialObservation } from "../../domain/artifactPartialObservation.js";
import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import { analyzeAndAdoptJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js";
-import { javascriptArtifactAnalysisScopes } from "./JavaScriptArtifactAnalysisTypes.js";
+import {
+ isSkippedJavaScriptAnalysis,
+ javascriptArtifactAnalysisScopes,
+} from "./JavaScriptArtifactAnalysisTypes.js";
import { createJavaScriptSourceAnalysis } from "../../artifacts/javascript/JavaScriptSourceAnalysis.js";
import { JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS } from "../../domain/javascript/javascriptAnalysisResourceControls.js";
import { createOwnedJavaScriptApplicationEvidenceCooperatively } from "./JavaScriptApplicationEvidence.js";
@@ -57,6 +60,7 @@ export interface JavaScriptArtifactReconstructionResult {
readonly text_bytes_read: number;
readonly invalid_utf8_files: number;
readonly parsed_javascript_files: number;
+ readonly skipped_javascript_files: number;
readonly visited_ast_nodes: number;
readonly findings: number;
readonly modules: number;
@@ -275,6 +279,9 @@ export const reconstructJavaScriptArtifact = async (
parsed_javascript_files: analysis.files.filter(
({ javascript }) => javascript !== null,
).length,
+ skipped_javascript_files: analysis.files.filter(
+ isSkippedJavaScriptAnalysis,
+ ).length,
visited_ast_nodes: analysis.visited_ast_nodes,
findings: analysis.findings,
modules: analysis.modules,
diff --git a/src/contracts/javascript/javascriptExportShapeComparisonExample.ts b/src/contracts/javascript/javascriptExportShapeComparisonExample.ts
index 65464336d..ddc6e47c3 100644
--- a/src/contracts/javascript/javascriptExportShapeComparisonExample.ts
+++ b/src/contracts/javascript/javascriptExportShapeComparisonExample.ts
@@ -71,6 +71,7 @@ const statistics = {
text_bytes_read: 60,
invalid_utf8_files: 0,
parsed_javascript_files: 1,
+ skipped_javascript_files: 0,
visited_ast_nodes: 12,
findings: 1,
modules: 0,
diff --git a/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts b/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts
index 4be73fab7..8a535c63f 100644
--- a/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts
+++ b/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts
@@ -166,6 +166,7 @@ export const JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE = createEvidence(
text_bytes_read: script.bytes,
invalid_utf8_files: 0,
parsed_javascript_files: 1,
+ skipped_javascript_files: 0,
visited_ast_nodes: 1,
findings: 0,
modules: 0,
diff --git a/src/domain/javascript/javascriptApplicationAnalysis.ts b/src/domain/javascript/javascriptApplicationAnalysis.ts
index 6387f8e90..7b1055b79 100644
--- a/src/domain/javascript/javascriptApplicationAnalysis.ts
+++ b/src/domain/javascript/javascriptApplicationAnalysis.ts
@@ -63,6 +63,10 @@ const reconstructionStatisticsSchema = z.strictObject({
text_bytes_read: countSchema,
invalid_utf8_files: countSchema,
parsed_javascript_files: countSchema,
+ // Old saved evidence predates this field: absence must stay unknown, not
+ // become a measured zero. The producer always writes it (required by the
+ // application-layer result type); readers treat a missing value as unknown.
+ skipped_javascript_files: countSchema.exactOptional(),
visited_ast_nodes: countSchema,
findings: countSchema,
modules: countSchema,
diff --git a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts
index a460de077..6356f92c0 100644
--- a/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts
+++ b/src/domain/javascript/javascriptApplicationAnalysisSteps.test.ts
@@ -324,3 +324,29 @@ describe("graphs transferred to their sealing steps", () => {
).toEqual(issuesFrom(() => createJavaScriptApplicationGraph(stale)));
});
});
+
+describe("skipped_javascript_files backward compatibility", () => {
+ it("preserves omission for saved evidence that predates the skipped counter", () => {
+ const legacy = structuredClone(
+ JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.normalized_result,
+ ) as Record<string, unknown>;
+ const statistics = legacy.statistics as Record<string, unknown>;
+ delete statistics.skipped_javascript_files;
+ const result = javascriptApplicationAnalysisResultSchema.parse(legacy);
+ // A pre-change result can contain the very resource-skipped files this
+ // field reports, but it has no counter: unknown must not become a
+ // measured zero that comparisons would treat as "nothing skipped".
+ expect("skipped_javascript_files" in result.statistics).toBe(false);
+ expect(result.statistics.skipped_javascript_files).toBeUndefined();
+ });
+
+ it("keeps the producer-written skipped count in current evidence", () => {
+ const current = structuredClone(
+ JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.normalized_result,
+ ) as Record<string, unknown>;
+ const statistics = current.statistics as Record<string, unknown>;
+ statistics.skipped_javascript_files = 3;
+ const result = javascriptApplicationAnalysisResultSchema.parse(current);
+ expect(result.statistics.skipped_javascript_files).toBe(3);
+ });
+});