Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): report files over 1 MB from ix map; --github pages past 100, stable patch id
KageBinary8494b701cf6f
diff --git a/ix-cli/src/cli/__tests__/github-fetch.test.ts b/ix-cli/src/cli/__tests__/github-fetch.test.tsindex c5ff4ec4..6ef269ea 100644--- a/ix-cli/src/cli/__tests__/github-fetch.test.ts+++ b/ix-cli/src/cli/__tests__/github-fetch.test.ts@@ -1,18 +1,42 @@// Copyright 2026 Ix Infrastructure Inc.-import { describe, it, expect } from "vitest";+import { describe, expect, it } from "vitest";-describe("parseGitHubRepo", () => {- it("parses owner/repo format", async () => {- const { parseGitHubRepo } = await import("../github/fetch.js");- expect(parseGitHubRepo("ix-infrastructure/IX-Memory")).toEqual({- owner: "ix-infrastructure",- repo: "IX-Memory",- });+import { ghFetchList } from "../github/fetch.js";++describe("ghFetchList", () => {+ /** A list endpoint holding `total` items, recording the URLs it was asked for. */+ const endpoint = (total: number) => {+ const urls: string[] = [];+ const fetchPage = async (url: string): Promise<number[]> => {+ urls.push(url);+ const q = new URL(url).searchParams;+ const perPage = Math.min(Number(q.get("per_page")), 100);+ const page = Number(q.get("page"));+ const start = (page - 1) * perPage;+ return Array.from({ length: Math.max(0, Math.min(perPage, total - start)) }, (_, i) => start + i);+ };+ return { urls, fetchPage };+ };++ it("pages past GitHub's 100-item cap up to the limit asked for", async () => {+ const { urls, fetchPage } = endpoint(1000);+ const items = await ghFetchList("https://api.github.com/repos/o/r/issues?state=all", "t", 250, fetchPage);+ expect(items).toEqual(Array.from({ length: 250 }, (_, i) => i));+ expect(urls).toEqual([+ "https://api.github.com/repos/o/r/issues?state=all&per_page=100&page=1",+ "https://api.github.com/repos/o/r/issues?state=all&per_page=100&page=2",+ "https://api.github.com/repos/o/r/issues?state=all&per_page=100&page=3",+ ]);});- it("throws on invalid format", async () => {- const { parseGitHubRepo } = await import("../github/fetch.js");- expect(() => parseGitHubRepo("invalid")).toThrow();+ it("stops at a short page, and asks for no more than it needs", async () => {+ const short = endpoint(130);+ expect(await ghFetchList("https://api.github.com/x", "t", 500, short.fetchPage)).toHaveLength(130);+ expect(short.urls).toHaveLength(2);++ const small = endpoint(1000);+ expect(await ghFetchList("https://api.github.com/x", "t", 30, small.fetchPage)).toHaveLength(30);+ expect(small.urls).toEqual(["https://api.github.com/x?per_page=30&page=1"]);});});diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.tsindex 831d103d..d046848b 100644--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts@@ -1082,6 +1082,17 @@ describe("ingestFiles against a fake backend", () => {});});+ it("counts a file over the size limit in the summary, so ix map can say so", async () => {+ fixture(2);+ writeFileSync(join(repo, "src", "huge.ts"), `export const big = "${"x".repeat(1024 * 1024 + 10)}";\n`, "utf8");+ execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });++ const summary = await run();++ expect(summary.filesTooLarge).toBe(1);+ expect(summary.patchesApplied).toBe(2);+ });+it("ingests only the languages --lang names, and keeps the rest of the baseline", async () => {fixture(3);writeFileSync(join(repo, "src", "tool.py"), "def tool():\n return 1\n", "utf8");diff --git a/ix-cli/src/cli/__tests__/map-baseline.test.ts b/ix-cli/src/cli/__tests__/map-baseline.test.tsindex 6b35410d..3d20789d 100644--- a/ix-cli/src/cli/__tests__/map-baseline.test.ts+++ b/ix-cli/src/cli/__tests__/map-baseline.test.ts@@ -6,7 +6,7 @@ import * as os from "node:os";import * as path from "node:path";import { persistIngestBaselineIfClean } from "../commands/ingest.js";-import { persistCompletedMapBaseline } from "../commands/map.js";+import { describeTooLargeFiles, persistCompletedMapBaseline } from "../commands/map.js";import { loadMapBaseline, saveMapBaseline } from "../map-baseline.js";import { ingestMtimeCachePath } from "../config.js";import { hasCompletedMapBaseline } from "../stale.js";@@ -181,3 +181,13 @@ describe("upgrade from a baseline written before the map marker", () => {expect(hasCompletedMapBaseline(root)).toBe(false);});});++describe("describeTooLargeFiles", () => {+ it("names files left out for size, and says nothing when there were none", () => {+ expect(describeTooLargeFiles({ filesTooLarge: 1 })).toBe("1 file over 1 MB was not mapped.");+ expect(describeTooLargeFiles({ filesTooLarge: 3 })).toBe("3 files over 1 MB were not mapped.");+ expect(describeTooLargeFiles({ filesTooLarge: 0 })).toBeUndefined();+ expect(describeTooLargeFiles({})).toBeUndefined();+ expect(describeTooLargeFiles(undefined)).toBeUndefined();+ });+});diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex b7c7e070..394dc44d 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -1100,6 +1100,8 @@ export interface IngestFilesSummary {* back out. That is the shape a #527 run has, and it is exactly right here.*/filesSkippedAsUnchanged: number;+ /** Files over the 1 MB parse limit, left out of the graph. Absent where no run counted them. */+ filesTooLarge?: number;parseErrors: number;commitErrors: number;stitchErrors: number;@@ -3982,6 +3984,7 @@ export async function ingestFiles(idempotentPatches,replayedChanges: [...replayedChanges].sort(),filesSkippedAsUnchanged,+ filesTooLarge: tooLarge,// `+ crashedParses()`, as the baseline and delete guards already do. Files// lost to a dead parse pool raise `filesSkippedUnparsed`, never// `parseErrors`, so without this everything downstream read the run as@@ -4303,7 +4306,10 @@ async function ingestGitHub(opts: {for (const commit of data.commits) allOps.push(...transformCommit(repo, commit));const patch: GraphPatchPayload = {- patchId: deterministicId(`github://${repo.owner}/${repo.repo}:${since}:${Date.now()}`),+ // From what is sent, not the clock: the same issues, PRs and commits give+ // the same id, so re-running an unchanged ingest is a no-op on the backend+ // instead of a new revision of identical facts.+ patchId: deterministicId(`github://${repo.owner}/${repo.repo}:${sha256(Buffer.from(JSON.stringify(allOps)))}`),// IX_PATCH_ACTOR="" lets a kOS cloud backend stamp the verified principal// (it 403s a non-empty actor that differs from it); see core-ingestion patchActor().actor: process.env.IX_PATCH_ACTOR ?? 'ix/github-ingest',diff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.tsindex bd2914b6..37fd14b2 100644--- a/ix-cli/src/cli/commands/map.ts+++ b/ix-cli/src/cli/commands/map.ts@@ -286,10 +286,25 @@ export function describeDroppedFiles(}function emitDroppedFileWarning(- ingest: Pick<IngestFilesSummary, "parseErrors" | "commitErrors"> | undefined,+ ingest: Pick<IngestFilesSummary, "parseErrors" | "commitErrors" | "filesTooLarge"> | undefined,): void {const message = describeDroppedFiles(ingest);if (message) process.stderr.write(chalk.yellow(` ${message}\n`));+ const tooLarge = describeTooLargeFiles(ingest);+ if (tooLarge) process.stderr.write(chalk.dim(` ${tooLarge}\n`));+}++/**+ * Files over the 1 MB parse limit are left out of the map. `ix ingest` lists+ * them in its summary; `ix map` said nothing, so a large generated or vendored+ * file was simply absent from the graph with no hint why.+ */+export function describeTooLargeFiles(+ ingest: Pick<IngestFilesSummary, "filesTooLarge"> | undefined,+): string | undefined {+ const n = ingest?.filesTooLarge ?? 0;+ if (n <= 0) return undefined;+ return `${n} file${n === 1 ? "" : "s"} over 1 MB ${n === 1 ? "was" : "were"} not mapped.`;}/**diff --git a/ix-cli/src/cli/github/fetch.ts b/ix-cli/src/cli/github/fetch.tsindex d5cfc9ca..f23d00e3 100644--- a/ix-cli/src/cli/github/fetch.ts+++ b/ix-cli/src/cli/github/fetch.ts@@ -84,6 +84,31 @@ async function ghFetch<T>(url: string, token: string): Promise<T> {return resp.json() as Promise<T>;}+/** GitHub serves at most this many items a page, whatever `per_page` asks for. */+const GITHUB_PAGE_MAX = 100;++/**+ * Up to `want` items of a list endpoint, page by page. `per_page` alone+ * silently capped every list at 100: `--limit 300` fetched 100 issues.+ * `url` carries its own query string, without `per_page` or `page`.+ */+export async function ghFetchList<T>(+ url: string,+ token: string,+ want: number,+ fetchPage: (url: string, token: string) => Promise<T[]> = ghFetch,+): Promise<T[]> {+ const perPage = Math.max(1, Math.min(want, GITHUB_PAGE_MAX));+ const out: T[] = [];+ for (let page = 1; out.length < want; page++) {+ const sep = url.includes("?") ? "&" : "?";+ const items = await fetchPage(`${url}${sep}per_page=${perPage}&page=${page}`, token);+ out.push(...items.slice(0, want - out.length));+ if (items.length < perPage) break;+ }+ return out;+}+export async function fetchGitHubData(repo: GitHubRepo,token: string,@@ -94,21 +119,23 @@ export async function fetchGitHubData(const limit = opts.limit ?? 50;const sinceParam = opts.since ? `&since=${encodeURIComponent(opts.since)}` : "";- const issues = await ghFetch<GitHubIssue[]>(- `${base}/issues?state=all&per_page=${limit}&sort=updated&direction=desc${sinceParam}`,- token+ const issues = await ghFetchList<GitHubIssue>(+ `${base}/issues?state=all&sort=updated&direction=desc${sinceParam}`,+ token,+ limit,);const realIssues = issues.filter((i: any) => !i.pull_request);- const pullRequests = await ghFetch<GitHubPR[]>(- `${base}/pulls?state=all&per_page=${limit}&sort=updated&direction=desc`,- token+ const pullRequests = await ghFetchList<GitHubPR>(+ `${base}/pulls?state=all&sort=updated&direction=desc`,+ token,+ limit,);- const commitLimit = Math.min(limit * 2, 100);- const commits = await ghFetch<GitHubCommit[]>(- `${base}/commits?per_page=${commitLimit}${sinceParam}`,- token+ const commits = await ghFetchList<GitHubCommit>(+ `${base}/commits?${sinceParam.slice(1)}`.replace(/\?$/, ""),+ token,+ limit * 2,);const issueComments = new Map<number, GitHubComment[]>();