Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

perf(javascript): index configuration and module lookups

morluto67252b23b9e3
diff --git a/src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts b/src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts
new file mode 100644
index 000000000..e9536443f
--- /dev/null
+++ b/src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts
@@ -0,0 +1,23 @@
+import { expect, it } from "vitest";
+
+import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js";
+
+it("keeps defaults linked to their own config reads across a large source", () => {
+ const count = 400;
+ const source = Array.from(
+ { length: count },
+ (_, index) =>
+ `const value${index} = process.env.KEY_${index} ?? "fallback-${index}";`,
+ ).join("\n");
+
+ const operations = analyzeJavaScriptSemantics(source).configurationOperations;
+ const defaults = operations.filter(({ kind }) => kind === "default");
+
+ expect(defaults).toHaveLength(count);
+ for (let index = 0; index < count; index += 1) {
+ expect(defaults.find(({ key }) => key === `KEY_${index}`)).toMatchObject({
+ key: `KEY_${index}`,
+ value: `fallback-${index}`,
+ });
+ }
+});
diff --git a/src/domain/javascript/javascriptSemanticDataEffects.ts b/src/domain/javascript/javascriptSemanticDataEffects.ts
index 6dad44bf0..18abeb532 100644
--- a/src/domain/javascript/javascriptSemanticDataEffects.ts
+++ b/src/domain/javascript/javascriptSemanticDataEffects.ts
@@ -163,15 +163,24 @@ const collectDefaults = (
configurations: readonly ConfigurationCandidate[],
): JavaScriptSemanticConfigurationOperation[] => {
const output: JavaScriptSemanticConfigurationOperation[] = [];
+ const configurationsByStart = configurations
+ .filter(
+ ({ node }) =>
+ node.start !== null &&
+ node.start !== undefined &&
+ node.end !== null &&
+ node.end !== undefined,
+ )
+ .toSorted(
+ (left, right) => (left.node.start ?? 0) - (right.node.start ?? 0),
+ );
traverseWithContext(program, context, (node) => {
if (
!t.isLogicalExpression(node) ||
(node.operator !== "??" && node.operator !== "||")
)
return;
- const sources = configurations.filter(({ node: source }) =>
- containsNode(node.left, source),
- );
+ const sources = configurationsIn(node.left, configurationsByStart);
if (sources.length !== 1 || sources[0] === undefined) return;
output.push({
configId: `config:default:${String(node.start ?? -1)}:${String(node.end ?? -1)}`,
@@ -188,6 +197,39 @@ const collectDefaults = (
return output;
};
+/** Find only configuration candidates whose source offsets can fit this left operand. */
+const configurationsIn = (
+ expression: t.Node,
+ candidates: readonly ConfigurationCandidate[],
+): readonly ConfigurationCandidate[] => {
+ if (
+ expression.start === null ||
+ expression.start === undefined ||
+ expression.end === null ||
+ expression.end === undefined
+ )
+ return [];
+ let low = 0;
+ let high = candidates.length;
+ while (low < high) {
+ const middle = low + Math.floor((high - low) / 2);
+ const start = candidates[middle]?.node.start;
+ if (start !== null && start !== undefined && start < expression.start)
+ low = middle + 1;
+ else high = middle;
+ }
+ const sources: ConfigurationCandidate[] = [];
+ for (let index = low; index < candidates.length; index += 1) {
+ const candidate = candidates[index];
+ if (candidate === undefined) continue;
+ const start = candidate.node.start;
+ if (start === null || start === undefined || start > expression.end) break;
+ if (containsNode(expression, candidate.node)) sources.push(candidate);
+ if (sources.length > 1) break;
+ }
+ return sources;
+};
+
const collectRequests = (
program: t.Program,
context: TraversalContext,
diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts
index 0a18ceb7d..af20eaa92 100644
--- a/src/domain/javascript/javascriptStaticAnalysis.ts
+++ b/src/domain/javascript/javascriptStaticAnalysis.ts
@@ -56,7 +56,7 @@ export const analyzeParsedJavaScriptStaticSource = (
file: ParsedJavaScriptSource,
openReceiverFacts?: ReadonlyMap<number, JavaScriptOpenReceiverFact>,
): JavaScriptStaticAnalysis => {
- const accumulator = createJavaScriptAnalysisAccumulator();
+ const accumulator = createJavaScriptAnalysisAccumulator(source.length);
traverseStaticSource(source, file, accumulator, openReceiverFacts);
addSourceMapDirectives(source, file.comments ?? [], accumulator);
return finalizeStaticAnalysis(source, file, accumulator);
@@ -105,41 +105,50 @@ const finalizeStaticAnalysis = (
visited_ast_nodes: accumulator.visitedNodes,
references: finalizeLocatedFindings(
accumulator.references,
- accumulator.modules,
+ accumulator.moduleRangeIndex,
),
endpoints: finalizeLocatedFindings(
accumulator.endpoints,
- accumulator.modules,
+ accumulator.moduleRangeIndex,
+ ),
+ storage: finalizeLocatedFindings(
+ accumulator.storage,
+ accumulator.moduleRangeIndex,
),
- storage: finalizeLocatedFindings(accumulator.storage, accumulator.modules),
bundler_registrations: sortedUnique(
accumulator.registrations,
registrationKey,
),
- role_paths: finalizeLocatedFindings(accumulator.roles, accumulator.modules),
+ role_paths: finalizeLocatedFindings(
+ accumulator.roles,
+ accumulator.moduleRangeIndex,
+ ),
source_map_urls: accumulator.sourceMaps,
vendors: detectVendors(source),
electron: {
browser_windows: finalizeLocatedFindings(
accumulator.browserWindows,
- accumulator.modules,
+ accumulator.moduleRangeIndex,
),
context_bridge_apis: finalizeLocatedFindings(
accumulator.contextBridgeApis,
- accumulator.modules,
+ accumulator.moduleRangeIndex,
+ ),
+ ipc: finalizeLocatedFindings(
+ accumulator.ipc,
+ accumulator.moduleRangeIndex,
),
- ipc: finalizeLocatedFindings(accumulator.ipc, accumulator.modules),
sender_validations: finalizeLocatedFindings(
accumulator.senderValidations,
- accumulator.modules,
+ accumulator.moduleRangeIndex,
),
utility_processes: finalizeLocatedFindings(
accumulator.utilityProcesses,
- accumulator.modules,
+ accumulator.moduleRangeIndex,
),
native_addon_bindings: finalizeLocatedFindings(
accumulator.nativeAddonBindings,
- accumulator.modules,
+ accumulator.moduleRangeIndex,
),
},
limitations,
diff --git a/src/domain/javascript/javascriptStaticAnalysisBundler.ts b/src/domain/javascript/javascriptStaticAnalysisBundler.ts
index 3e964f332..e82096f01 100644
--- a/src/domain/javascript/javascriptStaticAnalysisBundler.ts
+++ b/src/domain/javascript/javascriptStaticAnalysisBundler.ts
@@ -164,13 +164,15 @@ const recoverBundlerModule = (
const exportsValue = collectJavaScriptExports(factory);
const requireName = factoryRequireName(factory);
const asyncChunks = collectBundlerAsyncChunkKeys(factory, requireName);
- if (typeof factory.start === "number" && typeof factory.end === "number")
- accumulator.modules.push({
+ if (typeof factory.start === "number" && typeof factory.end === "number") {
+ const moduleRange = {
start: factory.start,
end: factory.end,
key,
requireName,
- });
+ };
+ accumulator.moduleRangeIndex.add(moduleRange);
+ }
return {
module: {
module_key: key,
diff --git a/src/domain/javascript/javascriptStaticAnalysisCalls.ts b/src/domain/javascript/javascriptStaticAnalysisCalls.ts
index 8addcb7cb..e984f77be 100644
--- a/src/domain/javascript/javascriptStaticAnalysisCalls.ts
+++ b/src/domain/javascript/javascriptStaticAnalysisCalls.ts
@@ -4,7 +4,6 @@ import { sanitizeEndpointCandidate } from "../browserObservation.js";
import {
addFindingOnce,
addLocatedFinding,
- moduleAtOffset,
} from "./javascriptStaticAnalysisFindings.js";
import {
STATIC_ROUTE_CALL_NAMES,
@@ -42,7 +41,7 @@ export const inspectCall = (
const { accumulator } = context;
const name = calleeName(node.callee);
const first = stringValue(node.arguments[0]);
- const module = moduleAtOffset(node.start, context.accumulator.modules);
+ const module = context.accumulator.moduleRangeIndex.find(node.start);
const moduleRequireName = module?.requireName ?? null;
const requireCall = requireCallSpecifier(node, name, moduleRequireName);
if (requireCall !== null)
diff --git a/src/domain/javascript/javascriptStaticAnalysisFindings.ts b/src/domain/javascript/javascriptStaticAnalysisFindings.ts
index 0def3a5bd..4108e7e72 100644
--- a/src/domain/javascript/javascriptStaticAnalysisFindings.ts
+++ b/src/domain/javascript/javascriptStaticAnalysisFindings.ts
@@ -6,6 +6,95 @@ import type {
LocatedJavaScriptFindingInput,
} from "./javascriptStaticAnalysisState.js";
+interface IndexedModuleRange {
+ readonly order: number;
+ readonly range: JavaScriptModuleRange;
+}
+
+interface ModuleRangeTreeNode {
+ left?: ModuleRangeTreeNode;
+ right?: ModuleRangeTreeNode;
+ best?: IndexedModuleRange;
+}
+
+/** Incremental point lookup preserving insertion-order precedence for overlaps. */
+export class JavaScriptModuleRangeIndex {
+ private readonly root: ModuleRangeTreeNode = {};
+ private readonly end: number;
+ private nextOrder = 0;
+
+ /** Create an empty offset index bounded by the analyzed source length. */
+ constructor(sourceLength: number) {
+ this.end = Math.max(1, sourceLength + 1);
+ }
+
+ /** Add a module range in the same discovery order used by the analysis pass. */
+ add(range: JavaScriptModuleRange): void {
+ const indexed = { order: this.nextOrder, range };
+ this.nextOrder += 1;
+ const start = Math.max(0, range.start);
+ const end = Math.min(this.end, range.end + 1);
+ if (start < end) this.insert(this.root, 0, this.end, start, end, indexed);
+ }
+
+ /** Return the first discovered inclusive range containing an offset. */
+ find(offset: number | null | undefined): JavaScriptModuleRange | undefined {
+ if (
+ offset === null ||
+ offset === undefined ||
+ !Number.isSafeInteger(offset) ||
+ offset < 0 ||
+ offset >= this.end
+ )
+ return undefined;
+ let node: ModuleRangeTreeNode | undefined = this.root;
+ let low = 0;
+ let high = this.end;
+ let best: IndexedModuleRange | undefined;
+ while (node !== undefined) {
+ if (
+ node.best !== undefined &&
+ (best === undefined || node.best.order < best.order)
+ )
+ best = node.best;
+ if (high - low <= 1) break;
+ const middle = low + Math.floor((high - low) / 2);
+ if (offset < middle) {
+ node = node.left;
+ high = middle;
+ } else {
+ node = node.right;
+ low = middle;
+ }
+ }
+ return best?.range;
+ }
+
+ private insert(
+ node: ModuleRangeTreeNode,
+ low: number,
+ high: number,
+ queryLow: number,
+ queryHigh: number,
+ value: IndexedModuleRange,
+ ): void {
+ if (queryLow <= low && high <= queryHigh) {
+ if (node.best === undefined || value.order < node.best.order)
+ node.best = value;
+ return;
+ }
+ const middle = low + Math.floor((high - low) / 2);
+ if (queryLow < middle) {
+ node.left ??= {};
+ this.insert(node.left, low, middle, queryLow, queryHigh, value);
+ }
+ if (queryHigh > middle) {
+ node.right ??= {};
+ this.insert(node.right, middle, high, queryLow, queryHigh, value);
+ }
+ }
+}
+
/** Add one module-attributable finding once per source location. */
export const addLocatedFinding = <
Value extends { readonly module_key: string | null },
@@ -39,18 +128,9 @@ export const finalizeLocatedFindings = <
Value extends { readonly module_key: string | null },
>(
values: readonly LocatedJavaScriptFinding<Value>[],
- modules: readonly JavaScriptModuleRange[],
+ modules: JavaScriptModuleRangeIndex,
): Value[] =>
values.map(({ offset, value }) => ({
...value,
- module_key: moduleAtOffset(offset, modules)?.key ?? null,
+ module_key: modules.find(offset)?.key ?? null,
}));
-
-/** Locate the recovered bundle factory containing one exact source offset. */
-export const moduleAtOffset = (
- offset: number | null | undefined,
- modules: readonly JavaScriptModuleRange[],
-): JavaScriptModuleRange | undefined => {
- if (offset === null || offset === undefined) return undefined;
- return modules.find(({ start, end }) => offset >= start && offset <= end);
-};
diff --git a/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts b/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts
new file mode 100644
index 000000000..075597a70
--- /dev/null
+++ b/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts
@@ -0,0 +1,69 @@
+import { expect, it } from "vitest";
+
+import { JavaScriptModuleRangeIndex } from "./javascriptStaticAnalysisFindings.js";
+import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js";
+
+it("preserves first insertion precedence and inclusive interval bounds", () => {
+ const nested = new JavaScriptModuleRangeIndex(100);
+ expect(nested.find(0)).toBeUndefined();
+ nested.add({ start: 10, end: 90, key: "outer-first", requireName: null });
+ nested.add({ start: 20, end: 30, key: "inner-second", requireName: null });
+ nested.add({ start: 30, end: 40, key: "later-overlap", requireName: null });
+ expect(nested.find(9)).toBeUndefined();
+ expect(nested.find(10)?.key).toBe("outer-first");
+ expect(nested.find(20)?.key).toBe("outer-first");
+ expect(nested.find(30)?.key).toBe("outer-first");
+ expect(nested.find(90)?.key).toBe("outer-first");
+ expect(nested.find(91)).toBeUndefined();
+ expect(nested.find(100)).toBeUndefined();
+});
+
+it("rejects missing and out-of-range lookup offsets", () => {
+ const index = new JavaScriptModuleRangeIndex(20);
+ index.add({ start: 5, end: 10, key: "bounded", requireName: null });
+ expect(index.find(undefined)).toBeUndefined();
+ expect(index.find(null)).toBeUndefined();
+ expect(index.find(-1)).toBeUndefined();
+ expect(index.find(0)).toBeUndefined();
+ expect(index.find(11)).toBeUndefined();
+ expect(index.find(20)).toBeUndefined();
+ expect(index.find(Number.NaN)).toBeUndefined();
+ expect(index.find(5)?.key).toBe("bounded");
+ expect(index.find(10)?.key).toBe("bounded");
+});
+
+it("keeps module references attributed across many real factory ranges", () => {
+ const count = 200;
+ const factories = Array.from({ length: count }, (_, index) =>
+ index === 0
+ ? '"module-0": function(module, exports, req) { globalThis.webpackChunkInner.push([[2], { "inner": function(innerModule, innerExports, req) { req("nested-dependency"); } }]); req("dependency-0"); }'
+ : `"module-${index}": function(module, exports, req) { req("dependency-${index}"); }`,
+ ).join(",\n");
+ const analysis = analyzeJavaScriptStaticSource(
+ `require("outside");\nglobalThis.webpackChunkApp.push([[1], {\n${factories}\n}]);`,
+ );
+ expect(analysis.references).toHaveLength(count + 2);
+ expect(
+ analysis.bundler_registrations.some(
+ ({ runtime, modules }) =>
+ runtime === "webpackChunkInner" &&
+ modules.some(({ module_key }) => module_key === "inner"),
+ ),
+ ).toBe(true);
+ expect(
+ analysis.references.find(({ specifier }) => specifier === "outside")
+ ?.module_key,
+ ).toBeNull();
+ for (let index = 0; index < count; index += 1) {
+ expect(
+ analysis.references.find(
+ ({ specifier }) => specifier === `dependency-${index}`,
+ )?.module_key,
+ ).toBe(`module-${index}`);
+ }
+ expect(
+ analysis.references.find(
+ ({ specifier }) => specifier === "nested-dependency",
+ )?.module_key,
+ ).toBe("module-0");
+});
diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts
index b830b1bc4..23de6952b 100644
--- a/src/domain/javascript/javascriptStaticAnalysisState.ts
+++ b/src/domain/javascript/javascriptStaticAnalysisState.ts
@@ -16,6 +16,7 @@ import type {
ElectronSenderValidationFinding,
ElectronUtilityProcessFinding,
} from "./electronStaticAnalysisTypes.js";
+import { JavaScriptModuleRangeIndex } from "./javascriptStaticAnalysisFindings.js";
/** Source offsets for one recovered bundle module factory. */
export interface JavaScriptModuleRange {
@@ -45,7 +46,7 @@ export interface JavaScriptAnalysisAccumulator {
readonly senderValidations: LocatedJavaScriptFinding<ElectronSenderValidationFinding>[];
readonly utilityProcesses: LocatedJavaScriptFinding<ElectronUtilityProcessFinding>[];
readonly nativeAddonBindings: LocatedJavaScriptFinding<ElectronNativeAddonBindingFinding>[];
- readonly modules: JavaScriptModuleRange[];
+ readonly moduleRangeIndex: JavaScriptModuleRangeIndex;
readonly seen: Set<string>;
visitedNodes: number;
unknownFindings: number;
@@ -92,22 +93,23 @@ export interface LocatedJavaScriptFindingInput<
}
/** Create isolated mutable state for one source analysis. */
-export const createJavaScriptAnalysisAccumulator =
- (): JavaScriptAnalysisAccumulator => ({
- references: [],
- endpoints: [],
- storage: [],
- roles: [],
- sourceMaps: [],
- registrations: [],
- browserWindows: [],
- contextBridgeApis: [],
- ipc: [],
- senderValidations: [],
- utilityProcesses: [],
- nativeAddonBindings: [],
- modules: [],
- seen: new Set(),
- visitedNodes: 0,
- unknownFindings: 0,
- });
+export const createJavaScriptAnalysisAccumulator = (
+ sourceLength: number,
+): JavaScriptAnalysisAccumulator => ({
+ references: [],
+ endpoints: [],
+ storage: [],
+ roles: [],
+ sourceMaps: [],
+ registrations: [],
+ browserWindows: [],
+ contextBridgeApis: [],
+ ipc: [],
+ senderValidations: [],
+ utilityProcesses: [],
+ nativeAddonBindings: [],
+ moduleRangeIndex: new JavaScriptModuleRangeIndex(sourceLength),
+ seen: new Set(),
+ visitedNodes: 0,
+ unknownFindings: 0,
+});