Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

feat(map): report files_too_large in JSON and llm output

KageBinary495fa1516aee
diff --git a/docs/llm-format.md b/docs/llm-format.md
index 8e2712f6..3c75d5c5 100644
--- a/docs/llm-format.md
+++ b/docs/llm-format.md
@@ -219,6 +219,26 @@ follows the header and the totals are floors.
## Examples
+`ix map`:
+
+```
+map files=412 regions=9 levels=2 rev=1043 outcome=full_local_completed files_too_large=2
+too_large path=data/fixtures/large.json
+too_large path=vendor/bundle.js
+region id=3f9c1a2b kind=system label=Ix level=2 files=412 children=2 cohesion=0.41 coupling=0 confidence=0.9
+region id=7d02e4c1 kind=subsystem label="Cli / Client" level=1 files=87 parent=3f9c1a2b cohesion=0.62 coupling=4.1 confidence=0.74 signals=imports,calls
+```
+
+The `map` record carries `parse_errors`, `commit_errors` and `files_too_large`
+only when they are non-zero, so a clean map says nothing about them.
+`files_too_large` counts the files over the 1 MB parse limit, which were left
+out of the graph. Up to 20 of them follow as `too_large` rows, sorted by path,
+before the regions. A count above the number of rows means the rest were not
+listed. `--format json` carries the same two things as `files_too_large`
+and `files_too_large_paths`, and has them on every map: zero and an empty list
+when no file was too large. `ix map --silent` adds `too_large=N` to its one
+line.
+
`ix stats`:
```
diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.ts
index 37248988..9840f7cd 100644
--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts
@@ -1165,6 +1165,7 @@ describe("ingestFiles against a fake backend", () => {
const summary = await run();
expect(summary.filesTooLarge).toBe(1);
+ expect(summary.filesTooLargePaths).toEqual(["src/huge.ts"]);
expect(summary.patchesApplied).toBe(2);
});
diff --git a/ix-cli/src/cli/__tests__/llm-renderers.test.ts b/ix-cli/src/cli/__tests__/llm-renderers.test.ts
index a8801a47..348cce72 100644
--- a/ix-cli/src/cli/__tests__/llm-renderers.test.ts
+++ b/ix-cli/src/cli/__tests__/llm-renderers.test.ts
@@ -82,6 +82,22 @@ describe("renderMapLlm", () => {
const noIngest = captureLog(() => renderMapLlm(result, []));
expect(noIngest[0]).toBe("map files=1 regions=0 levels=1 rev=1 outcome=ok");
});
+
+ it("carries the oversized-file count and a too_large row per listed path, before the regions", () => {
+ const result: MapResult = { file_count: 1, region_count: 1, levels: 1, map_rev: 1, outcome: "ok", regions: [], hierarchy: [] };
+
+ const lines = captureLog(() => renderMapLlm(result, [region({ id: "x", label: "X" })], {
+ parseErrors: 0, commitErrors: 0, filesTooLarge: 3, filesTooLargePaths: ["data/big.json", "vendor/my lib.js"],
+ }));
+ expect(lines[0]).toBe("map files=1 regions=1 levels=1 rev=1 outcome=ok files_too_large=3");
+ expect(lines[1]).toBe("too_large path=data/big.json");
+ expect(lines[2]).toBe('too_large path="vendor/my lib.js"');
+ expect(lines[3]).toMatch(/^region id=x /);
+
+ // None, none of it.
+ const clean = captureLog(() => renderMapLlm(result, [], { parseErrors: 0, commitErrors: 0, filesTooLarge: 0, filesTooLargePaths: [] }));
+ expect(clean).toEqual(["map files=1 regions=0 levels=1 rev=1 outcome=ok"]);
+ });
});
describe("renderSubsystemScoreLlm", () => {
diff --git a/ix-cli/src/cli/__tests__/map-baseline.test.ts b/ix-cli/src/cli/__tests__/map-baseline.test.ts
index 4965de50..e4515bb6 100644
--- a/ix-cli/src/cli/__tests__/map-baseline.test.ts
+++ b/ix-cli/src/cli/__tests__/map-baseline.test.ts
@@ -5,8 +5,8 @@ import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
-import { persistIngestBaselineIfClean } from "../commands/ingest.js";
-import { describeTooLargeFiles, persistCompletedMapBaseline, silentSummaryLine } from "../commands/map.js";
+import { FILES_TOO_LARGE_LISTED, listTooLargeFiles, persistIngestBaselineIfClean } from "../commands/ingest.js";
+import { describeTooLargeFiles, mapJsonPayload, persistCompletedMapBaseline, silentSummaryLine } from "../commands/map.js";
import { loadMapBaseline, saveMapBaseline } from "../map-baseline.js";
import { ingestMtimeCachePath } from "../config.js";
import { hasCompletedMapBaseline } from "../stale.js";
@@ -203,3 +203,41 @@ describe("silentSummaryLine", () => {
expect(line.split("\n")).toHaveLength(2);
});
});
+
+describe("mapJsonPayload", () => {
+ const result = { file_count: 3, region_count: 0, levels: 1, map_rev: 9, outcome: "ok", regions: [], hierarchy: [] };
+
+ it("carries files_too_large and up to 20 of their paths, always present", () => {
+ const payload = mapJsonPayload(result, [], {
+ parseErrors: 0, commitErrors: 0, filesTooLarge: 2, filesTooLargePaths: ["a/big.json", "b/huge.ts"],
+ });
+ expect(payload.files_too_large).toBe(2);
+ expect(payload.files_too_large_paths).toEqual(["a/big.json", "b/huge.ts"]);
+ // Additive: the fields that were there before are unchanged.
+ expect(payload).toMatchObject({ file_count: 3, region_count: 0, parse_errors: 0, commit_errors: 0, stitch_skipped: null, regions: [] });
+ });
+
+ it("reports zero and an empty list when nothing was too large, or nothing was ingested locally", () => {
+ const clean = mapJsonPayload(result, [], { parseErrors: 0, commitErrors: 0, filesTooLarge: 0, filesTooLargePaths: [] });
+ expect(clean.files_too_large).toBe(0);
+ expect(clean.files_too_large_paths).toEqual([]);
+ const remote = mapJsonPayload(result, [], undefined);
+ expect(remote.files_too_large).toBe(0);
+ expect(remote.files_too_large_paths).toEqual([]);
+ expect(JSON.parse(JSON.stringify(remote))).toHaveProperty("files_too_large_paths");
+ });
+});
+
+describe("listTooLargeFiles", () => {
+ it("lists only the oversized files, relative and sorted, and at most FILES_TOO_LARGE_LISTED", () => {
+ const record = new Map<string, "empty" | "tooLarge">();
+ for (let i = 29; i >= 0; i--) record.set(`/repo/gen/f${String(i).padStart(2, "0")}.json`, "tooLarge");
+ record.set("/repo/empty.ts", "empty");
+ const listed = listTooLargeFiles(record, p => p.replace("/repo/", ""));
+ expect(listed).toHaveLength(FILES_TOO_LARGE_LISTED);
+ expect(listed[0]).toBe("gen/f00.json");
+ expect(listed).toEqual([...listed].sort());
+ expect(listed).not.toContain("empty.ts");
+ expect(listTooLargeFiles(undefined, p => p)).toEqual([]);
+ });
+});
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 976171a5..2280fa61 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -755,6 +755,23 @@ const PAYLOAD_TOO_LARGE_PATTERNS = [
'maximal transaction size',
];
+/** How many oversized files a summary names; the count covers the rest. */
+export const FILES_TOO_LARGE_LISTED = 20;
+
+/**
+ * The oversized files in the stat loop's record, workspace-relative, sorted,
+ * and cut to `FILES_TOO_LARGE_LISTED`. Exported for tests.
+ */
+export function listTooLargeFiles(
+ record: ReadonlyMap<string, 'empty' | 'tooLarge'> | undefined,
+ toRelative: (absPath: string) => string,
+): string[] {
+ if (record === undefined) return [];
+ const paths: string[] = [];
+ for (const [path, why] of record) if (why === 'tooLarge') paths.push(toRelative(path));
+ return paths.sort().slice(0, FILES_TOO_LARGE_LISTED);
+}
+
export function isPayloadTooLargeError(err: unknown): boolean {
const responseError = err as { status?: unknown; statusCode?: unknown } | null;
if (responseError?.status === 413 || responseError?.statusCode === 413) return true;
@@ -1281,6 +1298,12 @@ export interface IngestFilesSummary {
filesSkippedAsUnchanged: number;
/** Files over the 1 MB parse limit, left out of the graph. Absent where no run counted them. */
filesTooLarge?: number;
+ /**
+ * Which files those were, workspace-relative and sorted, at most
+ * `FILES_TOO_LARGE_LISTED` of them: `filesTooLarge` is the whole count.
+ * Absent where no run counted them.
+ */
+ filesTooLargePaths?: string[];
parseErrors: number;
commitErrors: number;
stitchErrors: number;
@@ -2152,6 +2175,8 @@ export async function ingestFiles(
*/
const runDeadlineExpired = (): boolean => opts.deadlineSignal?.aborted === true;
let tooLarge = 0;
+ /** The stat loop's record of empty and oversized files, once it exists; `tooLarge` counts its oversized ones. */
+ let skippedForSize: ReadonlyMap<string, 'empty' | 'tooLarge'> | undefined;
let minifiedLikely = 0;
let outsideRoot = 0;
let latestRev = 0;
@@ -2388,6 +2413,7 @@ export async function ingestFiles(
* is precisely what the check that reads this exists to prevent.
*/
const accountedByStatLoop = new Map<string, 'empty' | 'tooLarge'>();
+ skippedForSize = accountedByStatLoop;
for (const filePath of filePaths) {
try {
const st = fs.statSync(filePath);
@@ -4399,6 +4425,7 @@ export async function ingestFiles(
replayedChanges: [...replayedChanges].sort(),
filesSkippedAsUnchanged,
filesTooLarge: tooLarge,
+ filesTooLargePaths: listTooLargeFiles(skippedForSize, toWorkspaceRelative),
// `+ crashedParses()`, as the baseline and delete guards already do. Files
// lost to a dead parse pool raise `filesSkippedUnparsed`, never
// `parseErrors`, so without this everything downstream read the run as
diff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.ts
index 3c7088d6..140dcf48 100644
--- a/ix-cli/src/cli/commands/map.ts
+++ b/ix-cli/src/cli/commands/map.ts
@@ -450,11 +450,61 @@ Examples:
});
}
+/** `ix map --format json`. Exported for tests. */
+export function mapJsonPayload(
+ result: MapView,
+ regions: MapRegion[],
+ localIngest: Pick<
+ IngestFilesSummary,
+ "parseErrors" | "commitErrors" | "stitchSkipped" | "stitchSkippedRule" | "filesTooLarge" | "filesTooLargePaths"
+ > | undefined,
+): Record<string, unknown> {
+ return {
+ file_count: result.file_count,
+ region_count: regions.length,
+ levels: result.levels,
+ map_rev: result.map_rev,
+ outcome: result.outcome,
+ // Always present so a consumer can branch on them without a key check.
+ // A dropped file is silent otherwise: the backend still answers with a
+ // completed outcome, so `outcome` alone cannot distinguish a whole map
+ // from one missing every file that failed to build a patch (#554).
+ parse_errors: localIngest?.parseErrors ?? 0,
+ commit_errors: localIngest?.commitErrors ?? 0,
+ // Ix#568. The whole reason this is reported at all is hooks that run
+ // `ix map` and read the machine output; leaving it only in
+ // `ix ingest --format json` puts it where those hooks never look.
+ // `?? null`, not left undefined: JSON.stringify drops an undefined
+ // value, so a consumer could not tell the field apart from an older
+ // CLI that never emitted it. Its siblings are always present too.
+ stitch_skipped: localIngest?.stitchSkipped ?? null,
+ stitch_skipped_rule: localIngest?.stitchSkippedRule ?? null,
+ // Files over the 1 MB parse limit, left out of the graph, and up to 20
+ // of their paths (the count is the whole number). On stderr and in the
+ // --silent line already; here for the consumers that read this payload.
+ // Always present, like their siblings.
+ files_too_large: localIngest?.filesTooLarge ?? 0,
+ files_too_large_paths: localIngest?.filesTooLargePaths ?? [],
+ regions: regions.map((r: any) => ({
+ label: r.label,
+ level: r.level,
+ files: r.file_count,
+ cohesion: roundFloat(r.cohesion),
+ coupling: roundFloat(r.external_coupling),
+ confidence: roundFloat(r.confidence),
+ signals: r.dominant_signals,
+ })),
+ };
+}
+
/** Flat one-record-per-line region listing with explicit parent= for the llm format. */
export function renderMapLlm(
result: MapView,
regions: MapRegion[],
- ingest?: Pick<IngestFilesSummary, "parseErrors" | "commitErrors" | "stitchSkipped" | "stitchSkippedRule">,
+ ingest?: Pick<
+ IngestFilesSummary,
+ "parseErrors" | "commitErrors" | "stitchSkipped" | "stitchSkippedRule" | "filesTooLarge" | "filesTooLargePaths"
+ >,
): void {
console.log(llmLine("map", [
["files", result.file_count],
@@ -474,7 +524,13 @@ export function renderMapLlm(
// where they never look.
["stitch_skipped", ingest?.stitchSkipped],
["stitch_skipped_rule", ingest?.stitchSkippedRule],
+ // Files over the 1 MB parse limit, left out of the graph. Only when there
+ // are some; the `too_large` rows after this name up to 20 of them.
+ ["files_too_large", ingest?.filesTooLarge ? ingest.filesTooLarge : undefined],
]));
+ for (const path of ingest?.filesTooLarge ? ingest.filesTooLargePaths ?? [] : []) {
+ console.log(llmLine("too_large", [["path", path]]));
+ }
for (const r of regions) {
console.log(llmLine("region", [
["id", llmShortId(r.id)],
@@ -1138,36 +1194,7 @@ async function runMapCommand(pathArg: string | undefined, opts: { format: string
if (minConf > 0) regions = regions.filter(r => r.confidence >= minConf);
if (opts.format === "json") {
- printJson({
- file_count: result.file_count,
- region_count: regions.length,
- levels: result.levels,
- map_rev: result.map_rev,
- outcome: result.outcome,
- // Always present so a consumer can branch on them without a key check.
- // A dropped file is silent otherwise: the backend still answers with a
- // completed outcome, so `outcome` alone cannot distinguish a whole map
- // from one missing every file that failed to build a patch (#554).
- parse_errors: localIngest?.parseErrors ?? 0,
- commit_errors: localIngest?.commitErrors ?? 0,
- // Ix#568. The whole reason this is reported at all is hooks that run
- // `ix map` and read the machine output; leaving it only in
- // `ix ingest --format json` puts it where those hooks never look.
- // `?? null`, not left undefined: JSON.stringify drops an undefined
- // value, so a consumer could not tell the field apart from an older
- // CLI that never emitted it. Its siblings are always present too.
- stitch_skipped: localIngest?.stitchSkipped ?? null,
- stitch_skipped_rule: localIngest?.stitchSkippedRule ?? null,
- regions: regions.map((r: any) => ({
- label: r.label,
- level: r.level,
- files: r.file_count,
- cohesion: roundFloat(r.cohesion),
- coupling: roundFloat(r.external_coupling),
- confidence: roundFloat(r.confidence),
- signals: r.dominant_signals,
- })),
- });
+ printJson(mapJsonPayload(result, regions, localIngest));
return;
}
if (opts.format === "llm") {