Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
feat(cli): summarize committed reference-source inventories (#2768)
To3akaRin93e29d242ae6
diff --git a/docs/cli.md b/docs/cli.mdindex 6d81ec7c1..c43ec2181 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -214,8 +214,36 @@ Import an older source tree to compare with the current artifact:```bashrea import-reference-source /absolute/path/to/source+rea import-reference-source /absolute/path/to/source --detail summary --outline-depth 2 --json```+The default (`--detail full`) returns the complete `HistoricalSourceGraph`.+Choose `--detail summary` for a `HistoricalSourceSummary` of that same committed+inventory: its `HistoricalSourceManifest` contains the full graph and root+digests, with provenance, VCS facts, languages, manifest paths, entry and+relationship counts, parse-failure counts, file/directory states, exclusions+and limitations. The summary is a presentation view, not a graph accepted by+source-to-bundle comparison; keep the full output for that workflow.++`--outline-depth` requires `--detail summary` and accepts a nonnegative safe+integer. Depth 1 shows root children, depth 2 (the default) also shows their+children, and depth 0 omits all outline entries. Paths retain canonical Unicode+code-point order. `outline.omitted_entry_count` counts only inventoried entries+hidden by the chosen depth; it does not count unexamined children of excluded or+unreadable directories, or change `inventory_state`. Exclusions and source+limitations remain separate from this presentation omission.++`file_sizes.known_bytes` is the exact sum of known inventoried file sizes as+decimal text, so large sums cannot lose integer precision. Unknown sizes count+in `file_sizes.unknown_file_count`; excluded or unexamined paths are not included+in that total. A zero known-byte sum therefore does not establish an empty source+tree. Full output retains individual file hashes, relationships and parse-failure+diagnostics. Summary mode performs the same import and cleanup, then projects+the committed result without rereading source files. It reduces the selected+output, not the amount of source examined. A shallow tree or many exclusions can+still produce a large summary; depth 0 removes the outline but preserves the+other facts. No fixed output-size or runtime improvement is promised.+The import records hashes and metadata for the supplied files, separatelyfrom observations of the current app. For a directory inside a Git checkout,it records the HEAD of the nearest enclosing repository; whether the workingdiff --git a/src/cli/utilityCommands.ts b/src/cli/utilityCommands.tsindex 2a9d05a4c..a5da57ac5 100644--- a/src/cli/utilityCommands.ts+++ b/src/cli/utilityCommands.ts@@ -2,6 +2,7 @@ import { z } from "incur";import type { DirectAnalysis } from "../composition/directAnalysis.js";import type { ReferenceSourcePolicy } from "../domain/referenceSourcePolicy.js";+import { createHistoricalSourceSummary } from "../domain/referenceSourceSummary.js";import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";import { importReferenceSource } from "../application/ReferenceSourceImport.js";@@ -241,11 +242,48 @@ const registerReferenceSourceCommand = (.string().describe("Local source directory to import as historical reference"),}),- run: ({ args }) =>+ options: z.object({+ detail: z+ .enum(["full", "summary"])+ .default("full")+ .describe(+ "Complete committed graph or compact historical-source summary",+ ),+ outlineDepth: z+ .number()+ .int()+ .nonnegative()+ .safe()+ .optional()+ .describe(+ "Summary outline path depth (default: 2; 0 omits entries); requires --detail summary",+ ),+ }),+ run: ({ args, options }) =>logCliCommand(logger,"import-reference-source",async () => {+ if (+ options.outlineDepth !== undefined &&+ options.detail !== "summary"+ )+ return {+ error: "Import failed",+ ...projectAnalysisError(+ new AnalysisInputError(+ "import-reference-source",+ { userMessage: "--outline-depth requires --detail summary" },+ [+ {+ path: ["outlineDepth"],+ reason: "invalid_value",+ message: "--outline-depth requires --detail summary",+ },+ ],+ ),+ ),+ };const config = parseConfig(environment);if (!config.ok)return {@@ -255,6 +293,9 @@ const registerReferenceSourceCommand = (return runReferenceSourceImport(args.root,config.value.referenceSourcePolicy,+ options.detail === "summary"+ ? (options.outlineDepth ?? 2)+ : undefined,);},isReferenceSourceImportCliFailure,@@ -265,6 +306,7 @@ const registerReferenceSourceCommand = (const runReferenceSourceImport = async (root: string,policy: ReferenceSourcePolicy,+ outlineDepth: number | undefined,): Promise<unknown> => {const resources = new ArtifactResourceScope();let outcome:@@ -346,7 +388,9 @@ const runReferenceSourceImport = async (cause: closeOutcome.cause,}),};- return imported.value;+ return outlineDepth === undefined+ ? imported.value+ : createHistoricalSourceSummary(imported.value, outlineDepth);};const mergeReferenceSourceCleanup = (diff --git a/src/domain/referenceSourceSummary.test.ts b/src/domain/referenceSourceSummary.test.tsnew file mode 100644index 000000000..a0d23b0e1--- /dev/null+++ b/src/domain/referenceSourceSummary.test.ts@@ -0,0 +1,104 @@+import { describe, expect, it } from "vitest";++import {+ createHistoricalSourceGraph,+ createHistoricalSourceManifest,+ type HistoricalSourceGraphInput,+} from "./referenceSourceGraph.js";+import { createHistoricalSourceSummary } from "./referenceSourceSummary.js";++const input = (+ entries: HistoricalSourceGraphInput["entries"],+): HistoricalSourceGraphInput => ({+ schema: "HistoricalSourceGraph",+ authority: "historical-reference",+ root_alias: "$REFERENCE_ROOT",+ inventory_state: "unknown",+ entries,+ relationships: [],+ parse_failures: [],+ exclusions: [],+ languages: [],+ manifests: [],+ vcs: null,+ provenance: { importer: "fixture", importer_version: null, caller: "test" },+ limitations: ["Inventory coverage is unknown"],+});++describe("committed historical-source summaries", () => {+ it("preserves unknown coverage, sizes, directories and missing symlinks without mutating the graph", () => {+ const graph = createHistoricalSourceGraph(+ input([+ {+ path: "a",+ kind: "directory",+ tree_state: "unknown",+ classifications: ["unknown"],+ limitations: ["Not enumerated"],+ },+ {+ path: "b",+ kind: "file",+ sha256: null,+ size: null,+ content_state: "unreadable",+ language: null,+ classifications: ["unknown"],+ limitations: ["Not readable"],+ },+ {+ path: "c",+ kind: "symlink",+ target: "missing",+ target_state: "missing",+ classifications: ["unknown"],+ limitations: [],+ },+ ]),+ );+ const original = structuredClone(graph);+ const summary = createHistoricalSourceSummary(graph, 1);+ expect(summary).toMatchObject({+ manifest: createHistoricalSourceManifest(original),+ inventory_state: "unknown",+ vcs: null,+ provenance: original.provenance,+ file_sizes: { known_bytes: "0", unknown_file_count: 1 },+ counts: { entries: 3, files: 1, directories: 1, symlinks: 1 },+ directory_tree_states: { unknown: 1, enumerated: 0 },+ file_content_states: { unreadable: 1, hashed: 0 },+ limitations: original.limitations,+ outline: {+ entries: [+ original.entries[0],+ expect.objectContaining({ size: null, content_state: "unreadable" }),+ original.entries[2],+ ],+ omitted_entry_count: 0,+ },+ });+ expect(summary.outline.entries[1]).not.toHaveProperty("sha256");+ expect(graph).toEqual(original);+ });++ it("sums known bytes losslessly beyond the JSON safe integer range", () => {+ const graph = createHistoricalSourceGraph(+ input(+ ["a", "b"].map((path) => ({+ path,+ kind: "file",+ sha256: "a".repeat(64),+ size: Number.MAX_SAFE_INTEGER,+ content_state: "hashed",+ language: null,+ classifications: ["unknown"],+ limitations: [],+ })),+ ),+ );+ expect(createHistoricalSourceSummary(graph, 0)).toMatchObject({+ file_sizes: { known_bytes: "18014398509481982", unknown_file_count: 0 },+ outline: { entries: [], omitted_entry_count: 2 },+ });+ });+});diff --git a/src/domain/referenceSourceSummary.ts b/src/domain/referenceSourceSummary.tsnew file mode 100644index 000000000..e7e99e3fa--- /dev/null+++ b/src/domain/referenceSourceSummary.ts@@ -0,0 +1,93 @@+import {+ createHistoricalSourceManifest,+ type HistoricalSourceGraph,+} from "./referenceSourceGraph.js";++type Entry = HistoricalSourceGraph["entries"][number];++const outlineEntry = (entry: Entry) => {+ if (entry.kind !== "file") return entry;+ return {+ path: entry.path,+ kind: entry.kind,+ language: entry.language,+ size: entry.size,+ content_state: entry.content_state,+ classifications: entry.classifications,+ limitations: entry.limitations,+ };+};++/** Project one committed inventory without rereading or reinterpreting source. */+export const createHistoricalSourceSummary = (+ graph: HistoricalSourceGraph,+ outlineDepth: number,+) => {+ const manifest = createHistoricalSourceManifest(graph);+ const counts = {+ entries: graph.entries.length,+ files: 0,+ directories: 0,+ symlinks: 0,+ relationships: graph.relationships.length,+ parse_failures: graph.parse_failures.length,+ exclusions: graph.exclusions.length,+ };+ const contentStates = {+ hashed: 0,+ "redacted-secret": 0,+ excluded: 0,+ unreadable: 0,+ unknown: 0,+ };+ const treeStates = {+ enumerated: 0,+ partial: 0,+ excluded: 0,+ unreadable: 0,+ unknown: 0,+ };+ let knownBytes = 0n;+ let unknownSizeFiles = 0;+ const outline: ReturnType<typeof outlineEntry>[] = [];+ for (const entry of graph.entries) {+ if (entry.kind === "file") {+ counts.files += 1;+ contentStates[entry.content_state] += 1;+ if (entry.size === null) unknownSizeFiles += 1;+ else knownBytes += BigInt(entry.size);+ } else if (entry.kind === "directory") {+ counts.directories += 1;+ treeStates[entry.tree_state] += 1;+ } else counts.symlinks += 1;+ // Canonical entries already use Unicode code-point order; filtering preserves it.+ if (entry.path.split("/").length <= outlineDepth)+ outline.push(outlineEntry(entry));+ }+ return {+ schema: "HistoricalSourceSummary" as const,+ authority: graph.authority,+ root_alias: graph.root_alias,+ manifest,+ inventory_state: graph.inventory_state,+ provenance: graph.provenance,+ vcs: graph.vcs,+ languages: graph.languages,+ manifests: graph.manifests,+ counts,+ file_content_states: contentStates,+ directory_tree_states: treeStates,+ file_sizes: {+ // Decimal text keeps large totals exact; this covers only known inventoried sizes.+ known_bytes: knownBytes.toString(),+ unknown_file_count: unknownSizeFiles,+ },+ exclusions: graph.exclusions,+ limitations: graph.limitations,+ outline: {+ depth: outlineDepth,+ entries: outline,+ omitted_entry_count: graph.entries.length - outline.length,+ },+ };+};diff --git a/tests/boundary/cli/referenceSourceSummary.test.ts b/tests/boundary/cli/referenceSourceSummary.test.tsnew file mode 100644index 000000000..aed8cb871--- /dev/null+++ b/tests/boundary/cli/referenceSourceSummary.test.ts@@ -0,0 +1,190 @@+import { mkdir, writeFile } from "node:fs/promises";+import { join } from "node:path";++import { describe, expect } from "vitest";++import {+ createHistoricalSourceManifest,+ parseHistoricalSourceGraph,+} from "../../../src/domain/referenceSourceGraph.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { cliTest } from "../../support/cli/cliFixture.js";++describe.skipIf(process.platform === "win32")(+ "compiled historical-source summaries",+ () => {+ cliTest(+ "selects a committed summary without changing complete output",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-reference-summary-");+ await mkdir(join(root, "src", "nested"), { recursive: true });+ await Promise.all([+ writeFile(join(root, "package.json"), '{"name":"fixture"}\n'),+ writeFile(+ join(root, "src", "main.js"),+ 'import "./nested/value.js";\n',+ ),+ writeFile(+ join(root, "src", "nested", "value.js"),+ "export const value = 1;\n",+ ),+ writeFile(join(root, "\uE000.js"), "export const first = 1;\n"),+ writeFile(join(root, "\u{10000}.js"), "export const last = 2;\n"),+ ]);+ const run = (...options: string[]) =>+ cli.run({+ arguments: ["import-reference-source", root, "--json", ...options],+ cwd: root,+ });+ const complete = await run();+ expect(complete.exitCode).toBe(0);+ const graph = parseHistoricalSourceGraph(complete.json);+ expect((await run("--detail", "full")).stdout).toBe(complete.stdout);+ for (const depth of [undefined, 0, 1, 10]) {+ const result = await run(+ "--detail",+ "summary",+ ...(depth === undefined ? [] : ["--outline-depth", String(depth)]),+ );+ expect(result.exitCode).toBe(0);+ const selected = graph.entries.filter(+ (entry) => entry.path.split("/").length <= (depth ?? 2),+ );+ expect(result.json).toMatchObject({+ schema: "HistoricalSourceSummary",+ authority: graph.authority,+ root_alias: graph.root_alias,+ manifest: createHistoricalSourceManifest(graph),+ inventory_state: graph.inventory_state,+ provenance: graph.provenance,+ vcs: graph.vcs,+ languages: graph.languages,+ manifests: graph.manifests,+ counts: {+ entries: 7,+ files: 5,+ directories: 2,+ symlinks: 0,+ relationships: 1,+ parse_failures: 0,+ exclusions: 0,+ },+ outline: {+ depth: depth ?? 2,+ entries: selected.map(({ path, kind }) =>+ expect.objectContaining({ path, kind }),+ ),+ omitted_entry_count: graph.entries.length - selected.length,+ },+ });+ expect(result.json).not.toHaveProperty("relationships");+ }+ },+ );++ cliTest(+ "retains exclusions and parse coverage independently of a hidden outline",+ async ({ cli }) => {+ const root = await createTestTempDirectory(+ "rea-reference-partial-summary-",+ );+ await Promise.all([+ writeFile(join(root, ".gitignore"), "ignored.txt\n"),+ writeFile(join(root, "ignored.txt"), "ignored\n"),+ writeFile(join(root, "broken.js"), "export const = ;\n"),+ ]);+ const result = await cli.run({+ arguments: [+ "import-reference-source",+ root,+ "--detail",+ "summary",+ "--outline-depth",+ "0",+ "--json",+ ],+ cwd: root,+ });+ expect(result.exitCode).toBe(0);+ expect(result.json).toMatchObject({+ inventory_state: "partial",+ counts: { parse_failures: 1, exclusions: 1 },+ exclusions: [+ {+ path: "ignored.txt",+ reason: "project-ignored",+ pattern: "ignored.txt",+ },+ ],+ outline: { depth: 0, entries: [], omitted_entry_count: 2 },+ });+ },+ );+ },+);++describe("compiled historical-source summary input boundaries", () => {+ cliTest(+ "documents detail and depth in public command help",+ async ({ cli }) => {+ const result = await cli.run({+ arguments: ["import-reference-source", "--help"],+ });+ expect(result.exitCode).toBe(0);+ expect(result.stdout).toContain("--detail");+ expect(result.stdout).toContain("--outline-depth");+ expect(result.stdout).toContain("requires --detail summary");+ },+ );++ for (const options of [+ ["--detail", "summry"],+ ["--detail", "summary", "--outline-depth", "-1"],+ ["--detail", "summary", "--outline-depth", "1.5"],+ ["--detail", "summary", "--outline-depth", "NaN"],+ ["--detail", "summary", "--outline-depth", "9007199254740992"],+ ["--outline-depth", "1"],+ ["--detail", "full", "--outline-depth", "1"],+ ]) {+ cliTest(`rejects ${options.join(" ")}`, async ({ cli }) => {+ const root = await createTestTempDirectory(+ "rea-reference-summary-input-",+ );+ const result = await cli.run({+ arguments: ["import-reference-source", root, ...options, "--json"],+ cwd: root,+ });+ expect(result.exitCode).toBe(1);+ expect(result.stdout).not.toContain("HistoricalSourceGraph");+ expect(result.stdout).not.toContain("HistoricalSourceSummary");+ expect(result.stdout).toMatch(/detail|outline/u);+ });+ }++ cliTest(+ "preserves the missing-root failure in summary mode",+ async ({ cli }) => {+ const root = await createTestTempDirectory(+ "rea-reference-summary-missing-",+ );+ const result = await cli.run({+ arguments: [+ "import-reference-source",+ join(root, "missing"),+ "--detail",+ "summary",+ "--json",+ ],+ cwd: root,+ });+ expect(result.exitCode).toBe(1);+ expect(result.json).toMatchObject({+ error: "Import failed",+ category: "invalid_input",+ message: expect.stringContaining(+ "Reference source root could not be resolved",+ ),+ });+ },+ );+});