Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

feat(cli): summarize committed reference-source inventories (#2768)

To3akaRin93e29d242ae6
diff --git a/docs/cli.md b/docs/cli.md
index 6d81ec7c1..c43ec2181 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -214,8 +214,36 @@ Import an older source tree to compare with the current artifact:
```bash
rea import-reference-source /absolute/path/to/source
+rea import-reference-source /absolute/path/to/source --detail summary --outline-depth 2 --json
```
+The default (`--detail full`) returns the complete `HistoricalSourceGraph`.
+Choose `--detail summary` for a `HistoricalSourceSummary` of that same committed
+inventory: its `HistoricalSourceManifest` contains the full graph and root
+digests, with provenance, VCS facts, languages, manifest paths, entry and
+relationship counts, parse-failure counts, file/directory states, exclusions
+and limitations. The summary is a presentation view, not a graph accepted by
+source-to-bundle comparison; keep the full output for that workflow.
+
+`--outline-depth` requires `--detail summary` and accepts a nonnegative safe
+integer. Depth 1 shows root children, depth 2 (the default) also shows their
+children, and depth 0 omits all outline entries. Paths retain canonical Unicode
+code-point order. `outline.omitted_entry_count` counts only inventoried entries
+hidden by the chosen depth; it does not count unexamined children of excluded or
+unreadable directories, or change `inventory_state`. Exclusions and source
+limitations remain separate from this presentation omission.
+
+`file_sizes.known_bytes` is the exact sum of known inventoried file sizes as
+decimal text, so large sums cannot lose integer precision. Unknown sizes count
+in `file_sizes.unknown_file_count`; excluded or unexamined paths are not included
+in that total. A zero known-byte sum therefore does not establish an empty source
+tree. Full output retains individual file hashes, relationships and parse-failure
+diagnostics. Summary mode performs the same import and cleanup, then projects
+the committed result without rereading source files. It reduces the selected
+output, not the amount of source examined. A shallow tree or many exclusions can
+still produce a large summary; depth 0 removes the outline but preserves the
+other facts. No fixed output-size or runtime improvement is promised.
+
The import records hashes and metadata for the supplied files, separately
from observations of the current app. For a directory inside a Git checkout,
it records the HEAD of the nearest enclosing repository; whether the working
diff --git a/src/cli/utilityCommands.ts b/src/cli/utilityCommands.ts
index 2a9d05a4c..a5da57ac5 100644
--- a/src/cli/utilityCommands.ts
+++ b/src/cli/utilityCommands.ts
@@ -2,6 +2,7 @@ import { z } from "incur";
import type { DirectAnalysis } from "../composition/directAnalysis.js";
import type { ReferenceSourcePolicy } from "../domain/referenceSourcePolicy.js";
+import { createHistoricalSourceSummary } from "../domain/referenceSourceSummary.js";
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import { importReferenceSource } from "../application/ReferenceSourceImport.js";
@@ -241,11 +242,48 @@ const registerReferenceSourceCommand = (
.string()
.describe("Local source directory to import as historical reference"),
}),
- run: ({ args }) =>
+ options: z.object({
+ detail: z
+ .enum(["full", "summary"])
+ .default("full")
+ .describe(
+ "Complete committed graph or compact historical-source summary",
+ ),
+ outlineDepth: z
+ .number()
+ .int()
+ .nonnegative()
+ .safe()
+ .optional()
+ .describe(
+ "Summary outline path depth (default: 2; 0 omits entries); requires --detail summary",
+ ),
+ }),
+ run: ({ args, options }) =>
logCliCommand(
logger,
"import-reference-source",
async () => {
+ if (
+ options.outlineDepth !== undefined &&
+ options.detail !== "summary"
+ )
+ return {
+ error: "Import failed",
+ ...projectAnalysisError(
+ new AnalysisInputError(
+ "import-reference-source",
+ { userMessage: "--outline-depth requires --detail summary" },
+ [
+ {
+ path: ["outlineDepth"],
+ reason: "invalid_value",
+ message: "--outline-depth requires --detail summary",
+ },
+ ],
+ ),
+ ),
+ };
const config = parseConfig(environment);
if (!config.ok)
return {
@@ -255,6 +293,9 @@ const registerReferenceSourceCommand = (
return runReferenceSourceImport(
args.root,
config.value.referenceSourcePolicy,
+ options.detail === "summary"
+ ? (options.outlineDepth ?? 2)
+ : undefined,
);
},
isReferenceSourceImportCliFailure,
@@ -265,6 +306,7 @@ const registerReferenceSourceCommand = (
const runReferenceSourceImport = async (
root: string,
policy: ReferenceSourcePolicy,
+ outlineDepth: number | undefined,
): Promise<unknown> => {
const resources = new ArtifactResourceScope();
let outcome:
@@ -346,7 +388,9 @@ const runReferenceSourceImport = async (
cause: closeOutcome.cause,
}),
};
- return imported.value;
+ return outlineDepth === undefined
+ ? imported.value
+ : createHistoricalSourceSummary(imported.value, outlineDepth);
};
const mergeReferenceSourceCleanup = (
diff --git a/src/domain/referenceSourceSummary.test.ts b/src/domain/referenceSourceSummary.test.ts
new file mode 100644
index 000000000..a0d23b0e1
--- /dev/null
+++ b/src/domain/referenceSourceSummary.test.ts
@@ -0,0 +1,104 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ createHistoricalSourceGraph,
+ createHistoricalSourceManifest,
+ type HistoricalSourceGraphInput,
+} from "./referenceSourceGraph.js";
+import { createHistoricalSourceSummary } from "./referenceSourceSummary.js";
+
+const input = (
+ entries: HistoricalSourceGraphInput["entries"],
+): HistoricalSourceGraphInput => ({
+ schema: "HistoricalSourceGraph",
+ authority: "historical-reference",
+ root_alias: "$REFERENCE_ROOT",
+ inventory_state: "unknown",
+ entries,
+ relationships: [],
+ parse_failures: [],
+ exclusions: [],
+ languages: [],
+ manifests: [],
+ vcs: null,
+ provenance: { importer: "fixture", importer_version: null, caller: "test" },
+ limitations: ["Inventory coverage is unknown"],
+});
+
+describe("committed historical-source summaries", () => {
+ it("preserves unknown coverage, sizes, directories and missing symlinks without mutating the graph", () => {
+ const graph = createHistoricalSourceGraph(
+ input([
+ {
+ path: "a",
+ kind: "directory",
+ tree_state: "unknown",
+ classifications: ["unknown"],
+ limitations: ["Not enumerated"],
+ },
+ {
+ path: "b",
+ kind: "file",
+ sha256: null,
+ size: null,
+ content_state: "unreadable",
+ language: null,
+ classifications: ["unknown"],
+ limitations: ["Not readable"],
+ },
+ {
+ path: "c",
+ kind: "symlink",
+ target: "missing",
+ target_state: "missing",
+ classifications: ["unknown"],
+ limitations: [],
+ },
+ ]),
+ );
+ const original = structuredClone(graph);
+ const summary = createHistoricalSourceSummary(graph, 1);
+ expect(summary).toMatchObject({
+ manifest: createHistoricalSourceManifest(original),
+ inventory_state: "unknown",
+ vcs: null,
+ provenance: original.provenance,
+ file_sizes: { known_bytes: "0", unknown_file_count: 1 },
+ counts: { entries: 3, files: 1, directories: 1, symlinks: 1 },
+ directory_tree_states: { unknown: 1, enumerated: 0 },
+ file_content_states: { unreadable: 1, hashed: 0 },
+ limitations: original.limitations,
+ outline: {
+ entries: [
+ original.entries[0],
+ expect.objectContaining({ size: null, content_state: "unreadable" }),
+ original.entries[2],
+ ],
+ omitted_entry_count: 0,
+ },
+ });
+ expect(summary.outline.entries[1]).not.toHaveProperty("sha256");
+ expect(graph).toEqual(original);
+ });
+
+ it("sums known bytes losslessly beyond the JSON safe integer range", () => {
+ const graph = createHistoricalSourceGraph(
+ input(
+ ["a", "b"].map((path) => ({
+ path,
+ kind: "file",
+ sha256: "a".repeat(64),
+ size: Number.MAX_SAFE_INTEGER,
+ content_state: "hashed",
+ language: null,
+ classifications: ["unknown"],
+ limitations: [],
+ })),
+ ),
+ );
+ expect(createHistoricalSourceSummary(graph, 0)).toMatchObject({
+ file_sizes: { known_bytes: "18014398509481982", unknown_file_count: 0 },
+ outline: { entries: [], omitted_entry_count: 2 },
+ });
+ });
+});
diff --git a/src/domain/referenceSourceSummary.ts b/src/domain/referenceSourceSummary.ts
new file mode 100644
index 000000000..e7e99e3fa
--- /dev/null
+++ b/src/domain/referenceSourceSummary.ts
@@ -0,0 +1,93 @@
+import {
+ createHistoricalSourceManifest,
+ type HistoricalSourceGraph,
+} from "./referenceSourceGraph.js";
+
+type Entry = HistoricalSourceGraph["entries"][number];
+
+const outlineEntry = (entry: Entry) => {
+ if (entry.kind !== "file") return entry;
+ return {
+ path: entry.path,
+ kind: entry.kind,
+ language: entry.language,
+ size: entry.size,
+ content_state: entry.content_state,
+ classifications: entry.classifications,
+ limitations: entry.limitations,
+ };
+};
+
+/** Project one committed inventory without rereading or reinterpreting source. */
+export const createHistoricalSourceSummary = (
+ graph: HistoricalSourceGraph,
+ outlineDepth: number,
+) => {
+ const manifest = createHistoricalSourceManifest(graph);
+ const counts = {
+ entries: graph.entries.length,
+ files: 0,
+ directories: 0,
+ symlinks: 0,
+ relationships: graph.relationships.length,
+ parse_failures: graph.parse_failures.length,
+ exclusions: graph.exclusions.length,
+ };
+ const contentStates = {
+ hashed: 0,
+ "redacted-secret": 0,
+ excluded: 0,
+ unreadable: 0,
+ unknown: 0,
+ };
+ const treeStates = {
+ enumerated: 0,
+ partial: 0,
+ excluded: 0,
+ unreadable: 0,
+ unknown: 0,
+ };
+ let knownBytes = 0n;
+ let unknownSizeFiles = 0;
+ const outline: ReturnType<typeof outlineEntry>[] = [];
+ for (const entry of graph.entries) {
+ if (entry.kind === "file") {
+ counts.files += 1;
+ contentStates[entry.content_state] += 1;
+ if (entry.size === null) unknownSizeFiles += 1;
+ else knownBytes += BigInt(entry.size);
+ } else if (entry.kind === "directory") {
+ counts.directories += 1;
+ treeStates[entry.tree_state] += 1;
+ } else counts.symlinks += 1;
+ // Canonical entries already use Unicode code-point order; filtering preserves it.
+ if (entry.path.split("/").length <= outlineDepth)
+ outline.push(outlineEntry(entry));
+ }
+ return {
+ schema: "HistoricalSourceSummary" as const,
+ authority: graph.authority,
+ root_alias: graph.root_alias,
+ manifest,
+ inventory_state: graph.inventory_state,
+ provenance: graph.provenance,
+ vcs: graph.vcs,
+ languages: graph.languages,
+ manifests: graph.manifests,
+ counts,
+ file_content_states: contentStates,
+ directory_tree_states: treeStates,
+ file_sizes: {
+ // Decimal text keeps large totals exact; this covers only known inventoried sizes.
+ known_bytes: knownBytes.toString(),
+ unknown_file_count: unknownSizeFiles,
+ },
+ exclusions: graph.exclusions,
+ limitations: graph.limitations,
+ outline: {
+ depth: outlineDepth,
+ entries: outline,
+ omitted_entry_count: graph.entries.length - outline.length,
+ },
+ };
+};
diff --git a/tests/boundary/cli/referenceSourceSummary.test.ts b/tests/boundary/cli/referenceSourceSummary.test.ts
new file mode 100644
index 000000000..aed8cb871
--- /dev/null
+++ b/tests/boundary/cli/referenceSourceSummary.test.ts
@@ -0,0 +1,190 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { describe, expect } from "vitest";
+
+import {
+ createHistoricalSourceManifest,
+ parseHistoricalSourceGraph,
+} from "../../../src/domain/referenceSourceGraph.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+describe.skipIf(process.platform === "win32")(
+ "compiled historical-source summaries",
+ () => {
+ cliTest(
+ "selects a committed summary without changing complete output",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-reference-summary-");
+ await mkdir(join(root, "src", "nested"), { recursive: true });
+ await Promise.all([
+ writeFile(join(root, "package.json"), '{"name":"fixture"}\n'),
+ writeFile(
+ join(root, "src", "main.js"),
+ 'import "./nested/value.js";\n',
+ ),
+ writeFile(
+ join(root, "src", "nested", "value.js"),
+ "export const value = 1;\n",
+ ),
+ writeFile(join(root, "\uE000.js"), "export const first = 1;\n"),
+ writeFile(join(root, "\u{10000}.js"), "export const last = 2;\n"),
+ ]);
+ const run = (...options: string[]) =>
+ cli.run({
+ arguments: ["import-reference-source", root, "--json", ...options],
+ cwd: root,
+ });
+ const complete = await run();
+ expect(complete.exitCode).toBe(0);
+ const graph = parseHistoricalSourceGraph(complete.json);
+ expect((await run("--detail", "full")).stdout).toBe(complete.stdout);
+ for (const depth of [undefined, 0, 1, 10]) {
+ const result = await run(
+ "--detail",
+ "summary",
+ ...(depth === undefined ? [] : ["--outline-depth", String(depth)]),
+ );
+ expect(result.exitCode).toBe(0);
+ const selected = graph.entries.filter(
+ (entry) => entry.path.split("/").length <= (depth ?? 2),
+ );
+ expect(result.json).toMatchObject({
+ schema: "HistoricalSourceSummary",
+ authority: graph.authority,
+ root_alias: graph.root_alias,
+ manifest: createHistoricalSourceManifest(graph),
+ inventory_state: graph.inventory_state,
+ provenance: graph.provenance,
+ vcs: graph.vcs,
+ languages: graph.languages,
+ manifests: graph.manifests,
+ counts: {
+ entries: 7,
+ files: 5,
+ directories: 2,
+ symlinks: 0,
+ relationships: 1,
+ parse_failures: 0,
+ exclusions: 0,
+ },
+ outline: {
+ depth: depth ?? 2,
+ entries: selected.map(({ path, kind }) =>
+ expect.objectContaining({ path, kind }),
+ ),
+ omitted_entry_count: graph.entries.length - selected.length,
+ },
+ });
+ expect(result.json).not.toHaveProperty("relationships");
+ }
+ },
+ );
+
+ cliTest(
+ "retains exclusions and parse coverage independently of a hidden outline",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory(
+ "rea-reference-partial-summary-",
+ );
+ await Promise.all([
+ writeFile(join(root, ".gitignore"), "ignored.txt\n"),
+ writeFile(join(root, "ignored.txt"), "ignored\n"),
+ writeFile(join(root, "broken.js"), "export const = ;\n"),
+ ]);
+ const result = await cli.run({
+ arguments: [
+ "import-reference-source",
+ root,
+ "--detail",
+ "summary",
+ "--outline-depth",
+ "0",
+ "--json",
+ ],
+ cwd: root,
+ });
+ expect(result.exitCode).toBe(0);
+ expect(result.json).toMatchObject({
+ inventory_state: "partial",
+ counts: { parse_failures: 1, exclusions: 1 },
+ exclusions: [
+ {
+ path: "ignored.txt",
+ reason: "project-ignored",
+ pattern: "ignored.txt",
+ },
+ ],
+ outline: { depth: 0, entries: [], omitted_entry_count: 2 },
+ });
+ },
+ );
+ },
+);
+
+describe("compiled historical-source summary input boundaries", () => {
+ cliTest(
+ "documents detail and depth in public command help",
+ async ({ cli }) => {
+ const result = await cli.run({
+ arguments: ["import-reference-source", "--help"],
+ });
+ expect(result.exitCode).toBe(0);
+ expect(result.stdout).toContain("--detail");
+ expect(result.stdout).toContain("--outline-depth");
+ expect(result.stdout).toContain("requires --detail summary");
+ },
+ );
+
+ for (const options of [
+ ["--detail", "summry"],
+ ["--detail", "summary", "--outline-depth", "-1"],
+ ["--detail", "summary", "--outline-depth", "1.5"],
+ ["--detail", "summary", "--outline-depth", "NaN"],
+ ["--detail", "summary", "--outline-depth", "9007199254740992"],
+ ["--outline-depth", "1"],
+ ["--detail", "full", "--outline-depth", "1"],
+ ]) {
+ cliTest(`rejects ${options.join(" ")}`, async ({ cli }) => {
+ const root = await createTestTempDirectory(
+ "rea-reference-summary-input-",
+ );
+ const result = await cli.run({
+ arguments: ["import-reference-source", root, ...options, "--json"],
+ cwd: root,
+ });
+ expect(result.exitCode).toBe(1);
+ expect(result.stdout).not.toContain("HistoricalSourceGraph");
+ expect(result.stdout).not.toContain("HistoricalSourceSummary");
+ expect(result.stdout).toMatch(/detail|outline/u);
+ });
+ }
+
+ cliTest(
+ "preserves the missing-root failure in summary mode",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory(
+ "rea-reference-summary-missing-",
+ );
+ const result = await cli.run({
+ arguments: [
+ "import-reference-source",
+ join(root, "missing"),
+ "--detail",
+ "summary",
+ "--json",
+ ],
+ cwd: root,
+ });
+ expect(result.exitCode).toBe(1);
+ expect(result.json).toMatchObject({
+ error: "Import failed",
+ category: "invalid_input",
+ message: expect.stringContaining(
+ "Reference source root could not be resolved",
+ ),
+ });
+ },
+ );
+});