Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
feat(map): report files_too_large in JSON and llm output
KageBinary495fa1516aee
diff --git a/docs/llm-format.md b/docs/llm-format.mdindex 8e2712f6..3c75d5c5 100644--- a/docs/llm-format.md+++ b/docs/llm-format.md@@ -219,6 +219,26 @@ follows the header and the totals are floors.## Examples+`ix map`:++```+map files=412 regions=9 levels=2 rev=1043 outcome=full_local_completed files_too_large=2+too_large path=data/fixtures/large.json+too_large path=vendor/bundle.js+region id=3f9c1a2b kind=system label=Ix level=2 files=412 children=2 cohesion=0.41 coupling=0 confidence=0.9+region id=7d02e4c1 kind=subsystem label="Cli / Client" level=1 files=87 parent=3f9c1a2b cohesion=0.62 coupling=4.1 confidence=0.74 signals=imports,calls+```++The `map` record carries `parse_errors`, `commit_errors` and `files_too_large`+only when they are non-zero, so a clean map says nothing about them.+`files_too_large` counts the files over the 1 MB parse limit, which were left+out of the graph. Up to 20 of them follow as `too_large` rows, sorted by path,+before the regions. A count above the number of rows means the rest were not+listed. `--format json` carries the same two things as `files_too_large`+and `files_too_large_paths`, and has them on every map: zero and an empty list+when no file was too large. `ix map --silent` adds `too_large=N` to its one+line.+`ix stats`:```diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.tsindex 37248988..9840f7cd 100644--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts@@ -1165,6 +1165,7 @@ describe("ingestFiles against a fake backend", () => {const summary = await run();expect(summary.filesTooLarge).toBe(1);+ expect(summary.filesTooLargePaths).toEqual(["src/huge.ts"]);expect(summary.patchesApplied).toBe(2);});diff --git a/ix-cli/src/cli/__tests__/llm-renderers.test.ts b/ix-cli/src/cli/__tests__/llm-renderers.test.tsindex a8801a47..348cce72 100644--- a/ix-cli/src/cli/__tests__/llm-renderers.test.ts+++ b/ix-cli/src/cli/__tests__/llm-renderers.test.ts@@ -82,6 +82,22 @@ describe("renderMapLlm", () => {const noIngest = captureLog(() => renderMapLlm(result, []));expect(noIngest[0]).toBe("map files=1 regions=0 levels=1 rev=1 outcome=ok");});++ it("carries the oversized-file count and a too_large row per listed path, before the regions", () => {+ const result: MapResult = { file_count: 1, region_count: 1, levels: 1, map_rev: 1, outcome: "ok", regions: [], hierarchy: [] };++ const lines = captureLog(() => renderMapLlm(result, [region({ id: "x", label: "X" })], {+ parseErrors: 0, commitErrors: 0, filesTooLarge: 3, filesTooLargePaths: ["data/big.json", "vendor/my lib.js"],+ }));+ expect(lines[0]).toBe("map files=1 regions=1 levels=1 rev=1 outcome=ok files_too_large=3");+ expect(lines[1]).toBe("too_large path=data/big.json");+ expect(lines[2]).toBe('too_large path="vendor/my lib.js"');+ expect(lines[3]).toMatch(/^region id=x /);++ // None, none of it.+ const clean = captureLog(() => renderMapLlm(result, [], { parseErrors: 0, commitErrors: 0, filesTooLarge: 0, filesTooLargePaths: [] }));+ expect(clean).toEqual(["map files=1 regions=0 levels=1 rev=1 outcome=ok"]);+ });});describe("renderSubsystemScoreLlm", () => {diff --git a/ix-cli/src/cli/__tests__/map-baseline.test.ts b/ix-cli/src/cli/__tests__/map-baseline.test.tsindex 4965de50..e4515bb6 100644--- a/ix-cli/src/cli/__tests__/map-baseline.test.ts+++ b/ix-cli/src/cli/__tests__/map-baseline.test.ts@@ -5,8 +5,8 @@ import * as fs from "node:fs";import * as os from "node:os";import * as path from "node:path";-import { persistIngestBaselineIfClean } from "../commands/ingest.js";-import { describeTooLargeFiles, persistCompletedMapBaseline, silentSummaryLine } from "../commands/map.js";+import { FILES_TOO_LARGE_LISTED, listTooLargeFiles, persistIngestBaselineIfClean } from "../commands/ingest.js";+import { describeTooLargeFiles, mapJsonPayload, persistCompletedMapBaseline, silentSummaryLine } from "../commands/map.js";import { loadMapBaseline, saveMapBaseline } from "../map-baseline.js";import { ingestMtimeCachePath } from "../config.js";import { hasCompletedMapBaseline } from "../stale.js";@@ -203,3 +203,41 @@ describe("silentSummaryLine", () => {expect(line.split("\n")).toHaveLength(2);});});++describe("mapJsonPayload", () => {+ const result = { file_count: 3, region_count: 0, levels: 1, map_rev: 9, outcome: "ok", regions: [], hierarchy: [] };++ it("carries files_too_large and up to 20 of their paths, always present", () => {+ const payload = mapJsonPayload(result, [], {+ parseErrors: 0, commitErrors: 0, filesTooLarge: 2, filesTooLargePaths: ["a/big.json", "b/huge.ts"],+ });+ expect(payload.files_too_large).toBe(2);+ expect(payload.files_too_large_paths).toEqual(["a/big.json", "b/huge.ts"]);+ // Additive: the fields that were there before are unchanged.+ expect(payload).toMatchObject({ file_count: 3, region_count: 0, parse_errors: 0, commit_errors: 0, stitch_skipped: null, regions: [] });+ });++ it("reports zero and an empty list when nothing was too large, or nothing was ingested locally", () => {+ const clean = mapJsonPayload(result, [], { parseErrors: 0, commitErrors: 0, filesTooLarge: 0, filesTooLargePaths: [] });+ expect(clean.files_too_large).toBe(0);+ expect(clean.files_too_large_paths).toEqual([]);+ const remote = mapJsonPayload(result, [], undefined);+ expect(remote.files_too_large).toBe(0);+ expect(remote.files_too_large_paths).toEqual([]);+ expect(JSON.parse(JSON.stringify(remote))).toHaveProperty("files_too_large_paths");+ });+});++describe("listTooLargeFiles", () => {+ it("lists only the oversized files, relative and sorted, and at most FILES_TOO_LARGE_LISTED", () => {+ const record = new Map<string, "empty" | "tooLarge">();+ for (let i = 29; i >= 0; i--) record.set(`/repo/gen/f${String(i).padStart(2, "0")}.json`, "tooLarge");+ record.set("/repo/empty.ts", "empty");+ const listed = listTooLargeFiles(record, p => p.replace("/repo/", ""));+ expect(listed).toHaveLength(FILES_TOO_LARGE_LISTED);+ expect(listed[0]).toBe("gen/f00.json");+ expect(listed).toEqual([...listed].sort());+ expect(listed).not.toContain("empty.ts");+ expect(listTooLargeFiles(undefined, p => p)).toEqual([]);+ });+});diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 976171a5..2280fa61 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -755,6 +755,23 @@ const PAYLOAD_TOO_LARGE_PATTERNS = ['maximal transaction size',];+/** How many oversized files a summary names; the count covers the rest. */+export const FILES_TOO_LARGE_LISTED = 20;++/**+ * The oversized files in the stat loop's record, workspace-relative, sorted,+ * and cut to `FILES_TOO_LARGE_LISTED`. Exported for tests.+ */+export function listTooLargeFiles(+ record: ReadonlyMap<string, 'empty' | 'tooLarge'> | undefined,+ toRelative: (absPath: string) => string,+): string[] {+ if (record === undefined) return [];+ const paths: string[] = [];+ for (const [path, why] of record) if (why === 'tooLarge') paths.push(toRelative(path));+ return paths.sort().slice(0, FILES_TOO_LARGE_LISTED);+}+export function isPayloadTooLargeError(err: unknown): boolean {const responseError = err as { status?: unknown; statusCode?: unknown } | null;if (responseError?.status === 413 || responseError?.statusCode === 413) return true;@@ -1281,6 +1298,12 @@ export interface IngestFilesSummary {filesSkippedAsUnchanged: number;/** Files over the 1 MB parse limit, left out of the graph. Absent where no run counted them. */filesTooLarge?: number;+ /**+ * Which files those were, workspace-relative and sorted, at most+ * `FILES_TOO_LARGE_LISTED` of them: `filesTooLarge` is the whole count.+ * Absent where no run counted them.+ */+ filesTooLargePaths?: string[];parseErrors: number;commitErrors: number;stitchErrors: number;@@ -2152,6 +2175,8 @@ export async function ingestFiles(*/const runDeadlineExpired = (): boolean => opts.deadlineSignal?.aborted === true;let tooLarge = 0;+ /** The stat loop's record of empty and oversized files, once it exists; `tooLarge` counts its oversized ones. */+ let skippedForSize: ReadonlyMap<string, 'empty' | 'tooLarge'> | undefined;let minifiedLikely = 0;let outsideRoot = 0;let latestRev = 0;@@ -2388,6 +2413,7 @@ export async function ingestFiles(* is precisely what the check that reads this exists to prevent.*/const accountedByStatLoop = new Map<string, 'empty' | 'tooLarge'>();+ skippedForSize = accountedByStatLoop;for (const filePath of filePaths) {try {const st = fs.statSync(filePath);@@ -4399,6 +4425,7 @@ export async function ingestFiles(replayedChanges: [...replayedChanges].sort(),filesSkippedAsUnchanged,filesTooLarge: tooLarge,+ filesTooLargePaths: listTooLargeFiles(skippedForSize, toWorkspaceRelative),// `+ crashedParses()`, as the baseline and delete guards already do. Files// lost to a dead parse pool raise `filesSkippedUnparsed`, never// `parseErrors`, so without this everything downstream read the run asdiff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.tsindex 3c7088d6..140dcf48 100644--- a/ix-cli/src/cli/commands/map.ts+++ b/ix-cli/src/cli/commands/map.ts@@ -450,11 +450,61 @@ Examples:});}+/** `ix map --format json`. Exported for tests. */+export function mapJsonPayload(+ result: MapView,+ regions: MapRegion[],+ localIngest: Pick<+ IngestFilesSummary,+ "parseErrors" | "commitErrors" | "stitchSkipped" | "stitchSkippedRule" | "filesTooLarge" | "filesTooLargePaths"+ > | undefined,+): Record<string, unknown> {+ return {+ file_count: result.file_count,+ region_count: regions.length,+ levels: result.levels,+ map_rev: result.map_rev,+ outcome: result.outcome,+ // Always present so a consumer can branch on them without a key check.+ // A dropped file is silent otherwise: the backend still answers with a+ // completed outcome, so `outcome` alone cannot distinguish a whole map+ // from one missing every file that failed to build a patch (#554).+ parse_errors: localIngest?.parseErrors ?? 0,+ commit_errors: localIngest?.commitErrors ?? 0,+ // Ix#568. The whole reason this is reported at all is hooks that run+ // `ix map` and read the machine output; leaving it only in+ // `ix ingest --format json` puts it where those hooks never look.+ // `?? null`, not left undefined: JSON.stringify drops an undefined+ // value, so a consumer could not tell the field apart from an older+ // CLI that never emitted it. Its siblings are always present too.+ stitch_skipped: localIngest?.stitchSkipped ?? null,+ stitch_skipped_rule: localIngest?.stitchSkippedRule ?? null,+ // Files over the 1 MB parse limit, left out of the graph, and up to 20+ // of their paths (the count is the whole number). On stderr and in the+ // --silent line already; here for the consumers that read this payload.+ // Always present, like their siblings.+ files_too_large: localIngest?.filesTooLarge ?? 0,+ files_too_large_paths: localIngest?.filesTooLargePaths ?? [],+ regions: regions.map((r: any) => ({+ label: r.label,+ level: r.level,+ files: r.file_count,+ cohesion: roundFloat(r.cohesion),+ coupling: roundFloat(r.external_coupling),+ confidence: roundFloat(r.confidence),+ signals: r.dominant_signals,+ })),+ };+}+/** Flat one-record-per-line region listing with explicit parent= for the llm format. */export function renderMapLlm(result: MapView,regions: MapRegion[],- ingest?: Pick<IngestFilesSummary, "parseErrors" | "commitErrors" | "stitchSkipped" | "stitchSkippedRule">,+ ingest?: Pick<+ IngestFilesSummary,+ "parseErrors" | "commitErrors" | "stitchSkipped" | "stitchSkippedRule" | "filesTooLarge" | "filesTooLargePaths"+ >,): void {console.log(llmLine("map", [["files", result.file_count],@@ -474,7 +524,13 @@ export function renderMapLlm(// where they never look.["stitch_skipped", ingest?.stitchSkipped],["stitch_skipped_rule", ingest?.stitchSkippedRule],+ // Files over the 1 MB parse limit, left out of the graph. Only when there+ // are some; the `too_large` rows after this name up to 20 of them.+ ["files_too_large", ingest?.filesTooLarge ? ingest.filesTooLarge : undefined],]));+ for (const path of ingest?.filesTooLarge ? ingest.filesTooLargePaths ?? [] : []) {+ console.log(llmLine("too_large", [["path", path]]));+ }for (const r of regions) {console.log(llmLine("region", [["id", llmShortId(r.id)],@@ -1138,36 +1194,7 @@ async function runMapCommand(pathArg: string | undefined, opts: { format: stringif (minConf > 0) regions = regions.filter(r => r.confidence >= minConf);if (opts.format === "json") {- printJson({- file_count: result.file_count,- region_count: regions.length,- levels: result.levels,- map_rev: result.map_rev,- outcome: result.outcome,- // Always present so a consumer can branch on them without a key check.- // A dropped file is silent otherwise: the backend still answers with a- // completed outcome, so `outcome` alone cannot distinguish a whole map- // from one missing every file that failed to build a patch (#554).- parse_errors: localIngest?.parseErrors ?? 0,- commit_errors: localIngest?.commitErrors ?? 0,- // Ix#568. The whole reason this is reported at all is hooks that run- // `ix map` and read the machine output; leaving it only in- // `ix ingest --format json` puts it where those hooks never look.- // `?? null`, not left undefined: JSON.stringify drops an undefined- // value, so a consumer could not tell the field apart from an older- // CLI that never emitted it. Its siblings are always present too.- stitch_skipped: localIngest?.stitchSkipped ?? null,- stitch_skipped_rule: localIngest?.stitchSkippedRule ?? null,- regions: regions.map((r: any) => ({- label: r.label,- level: r.level,- files: r.file_count,- cohesion: roundFloat(r.cohesion),- coupling: roundFloat(r.external_coupling),- confidence: roundFloat(r.confidence),- signals: r.dominant_signals,- })),- });+ printJson(mapJsonPayload(result, regions, localIngest));return;}if (opts.format === "llm") {