Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): report files over 1 MB from ix map; --github pages past 100, stable patch id

KageBinary8494b701cf6f
diff --git a/ix-cli/src/cli/__tests__/github-fetch.test.ts b/ix-cli/src/cli/__tests__/github-fetch.test.ts
index c5ff4ec4..6ef269ea 100644
--- a/ix-cli/src/cli/__tests__/github-fetch.test.ts
+++ b/ix-cli/src/cli/__tests__/github-fetch.test.ts
@@ -1,18 +1,42 @@
// Copyright 2026 Ix Infrastructure Inc.
-import { describe, it, expect } from "vitest";
+import { describe, expect, it } from "vitest";
-describe("parseGitHubRepo", () => {
- it("parses owner/repo format", async () => {
- const { parseGitHubRepo } = await import("../github/fetch.js");
- expect(parseGitHubRepo("ix-infrastructure/IX-Memory")).toEqual({
- owner: "ix-infrastructure",
- repo: "IX-Memory",
- });
+import { ghFetchList } from "../github/fetch.js";
+
+describe("ghFetchList", () => {
+ /** A list endpoint holding `total` items, recording the URLs it was asked for. */
+ const endpoint = (total: number) => {
+ const urls: string[] = [];
+ const fetchPage = async (url: string): Promise<number[]> => {
+ urls.push(url);
+ const q = new URL(url).searchParams;
+ const perPage = Math.min(Number(q.get("per_page")), 100);
+ const page = Number(q.get("page"));
+ const start = (page - 1) * perPage;
+ return Array.from({ length: Math.max(0, Math.min(perPage, total - start)) }, (_, i) => start + i);
+ };
+ return { urls, fetchPage };
+ };
+
+ it("pages past GitHub's 100-item cap up to the limit asked for", async () => {
+ const { urls, fetchPage } = endpoint(1000);
+ const items = await ghFetchList("https://api.github.com/repos/o/r/issues?state=all", "t", 250, fetchPage);
+ expect(items).toEqual(Array.from({ length: 250 }, (_, i) => i));
+ expect(urls).toEqual([
+ "https://api.github.com/repos/o/r/issues?state=all&per_page=100&page=1",
+ "https://api.github.com/repos/o/r/issues?state=all&per_page=100&page=2",
+ "https://api.github.com/repos/o/r/issues?state=all&per_page=100&page=3",
+ ]);
});
- it("throws on invalid format", async () => {
- const { parseGitHubRepo } = await import("../github/fetch.js");
- expect(() => parseGitHubRepo("invalid")).toThrow();
+ it("stops at a short page, and asks for no more than it needs", async () => {
+ const short = endpoint(130);
+ expect(await ghFetchList("https://api.github.com/x", "t", 500, short.fetchPage)).toHaveLength(130);
+ expect(short.urls).toHaveLength(2);
+
+ const small = endpoint(1000);
+ expect(await ghFetchList("https://api.github.com/x", "t", 30, small.fetchPage)).toHaveLength(30);
+ expect(small.urls).toEqual(["https://api.github.com/x?per_page=30&page=1"]);
});
});
diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.ts
index 831d103d..d046848b 100644
--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts
@@ -1082,6 +1082,17 @@ describe("ingestFiles against a fake backend", () => {
});
});
+ it("counts a file over the size limit in the summary, so ix map can say so", async () => {
+ fixture(2);
+ writeFileSync(join(repo, "src", "huge.ts"), `export const big = "${"x".repeat(1024 * 1024 + 10)}";\n`, "utf8");
+ execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });
+
+ const summary = await run();
+
+ expect(summary.filesTooLarge).toBe(1);
+ expect(summary.patchesApplied).toBe(2);
+ });
+
it("ingests only the languages --lang names, and keeps the rest of the baseline", async () => {
fixture(3);
writeFileSync(join(repo, "src", "tool.py"), "def tool():\n return 1\n", "utf8");
diff --git a/ix-cli/src/cli/__tests__/map-baseline.test.ts b/ix-cli/src/cli/__tests__/map-baseline.test.ts
index 6b35410d..3d20789d 100644
--- a/ix-cli/src/cli/__tests__/map-baseline.test.ts
+++ b/ix-cli/src/cli/__tests__/map-baseline.test.ts
@@ -6,7 +6,7 @@ import * as os from "node:os";
import * as path from "node:path";
import { persistIngestBaselineIfClean } from "../commands/ingest.js";
-import { persistCompletedMapBaseline } from "../commands/map.js";
+import { describeTooLargeFiles, persistCompletedMapBaseline } from "../commands/map.js";
import { loadMapBaseline, saveMapBaseline } from "../map-baseline.js";
import { ingestMtimeCachePath } from "../config.js";
import { hasCompletedMapBaseline } from "../stale.js";
@@ -181,3 +181,13 @@ describe("upgrade from a baseline written before the map marker", () => {
expect(hasCompletedMapBaseline(root)).toBe(false);
});
});
+
+describe("describeTooLargeFiles", () => {
+ it("names files left out for size, and says nothing when there were none", () => {
+ expect(describeTooLargeFiles({ filesTooLarge: 1 })).toBe("1 file over 1 MB was not mapped.");
+ expect(describeTooLargeFiles({ filesTooLarge: 3 })).toBe("3 files over 1 MB were not mapped.");
+ expect(describeTooLargeFiles({ filesTooLarge: 0 })).toBeUndefined();
+ expect(describeTooLargeFiles({})).toBeUndefined();
+ expect(describeTooLargeFiles(undefined)).toBeUndefined();
+ });
+});
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index b7c7e070..394dc44d 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -1100,6 +1100,8 @@ export interface IngestFilesSummary {
* back out. That is the shape a #527 run has, and it is exactly right here.
*/
filesSkippedAsUnchanged: number;
+ /** Files over the 1 MB parse limit, left out of the graph. Absent where no run counted them. */
+ filesTooLarge?: number;
parseErrors: number;
commitErrors: number;
stitchErrors: number;
@@ -3982,6 +3984,7 @@ export async function ingestFiles(
idempotentPatches,
replayedChanges: [...replayedChanges].sort(),
filesSkippedAsUnchanged,
+ filesTooLarge: tooLarge,
// `+ crashedParses()`, as the baseline and delete guards already do. Files
// lost to a dead parse pool raise `filesSkippedUnparsed`, never
// `parseErrors`, so without this everything downstream read the run as
@@ -4303,7 +4306,10 @@ async function ingestGitHub(opts: {
for (const commit of data.commits) allOps.push(...transformCommit(repo, commit));
const patch: GraphPatchPayload = {
- patchId: deterministicId(`github://${repo.owner}/${repo.repo}:${since}:${Date.now()}`),
+ // From what is sent, not the clock: the same issues, PRs and commits give
+ // the same id, so re-running an unchanged ingest is a no-op on the backend
+ // instead of a new revision of identical facts.
+ patchId: deterministicId(`github://${repo.owner}/${repo.repo}:${sha256(Buffer.from(JSON.stringify(allOps)))}`),
// IX_PATCH_ACTOR="" lets a kOS cloud backend stamp the verified principal
// (it 403s a non-empty actor that differs from it); see core-ingestion patchActor().
actor: process.env.IX_PATCH_ACTOR ?? 'ix/github-ingest',
diff --git a/ix-cli/src/cli/commands/map.ts b/ix-cli/src/cli/commands/map.ts
index bd2914b6..37fd14b2 100644
--- a/ix-cli/src/cli/commands/map.ts
+++ b/ix-cli/src/cli/commands/map.ts
@@ -286,10 +286,25 @@ export function describeDroppedFiles(
}
function emitDroppedFileWarning(
- ingest: Pick<IngestFilesSummary, "parseErrors" | "commitErrors"> | undefined,
+ ingest: Pick<IngestFilesSummary, "parseErrors" | "commitErrors" | "filesTooLarge"> | undefined,
): void {
const message = describeDroppedFiles(ingest);
if (message) process.stderr.write(chalk.yellow(` ${message}\n`));
+ const tooLarge = describeTooLargeFiles(ingest);
+ if (tooLarge) process.stderr.write(chalk.dim(` ${tooLarge}\n`));
+}
+
+/**
+ * Files over the 1 MB parse limit are left out of the map. `ix ingest` lists
+ * them in its summary; `ix map` said nothing, so a large generated or vendored
+ * file was simply absent from the graph with no hint why.
+ */
+export function describeTooLargeFiles(
+ ingest: Pick<IngestFilesSummary, "filesTooLarge"> | undefined,
+): string | undefined {
+ const n = ingest?.filesTooLarge ?? 0;
+ if (n <= 0) return undefined;
+ return `${n} file${n === 1 ? "" : "s"} over 1 MB ${n === 1 ? "was" : "were"} not mapped.`;
}
/**
diff --git a/ix-cli/src/cli/github/fetch.ts b/ix-cli/src/cli/github/fetch.ts
index d5cfc9ca..f23d00e3 100644
--- a/ix-cli/src/cli/github/fetch.ts
+++ b/ix-cli/src/cli/github/fetch.ts
@@ -84,6 +84,31 @@ async function ghFetch<T>(url: string, token: string): Promise<T> {
return resp.json() as Promise<T>;
}
+/** GitHub serves at most this many items a page, whatever `per_page` asks for. */
+const GITHUB_PAGE_MAX = 100;
+
+/**
+ * Up to `want` items of a list endpoint, page by page. `per_page` alone
+ * silently capped every list at 100: `--limit 300` fetched 100 issues.
+ * `url` carries its own query string, without `per_page` or `page`.
+ */
+export async function ghFetchList<T>(
+ url: string,
+ token: string,
+ want: number,
+ fetchPage: (url: string, token: string) => Promise<T[]> = ghFetch,
+): Promise<T[]> {
+ const perPage = Math.max(1, Math.min(want, GITHUB_PAGE_MAX));
+ const out: T[] = [];
+ for (let page = 1; out.length < want; page++) {
+ const sep = url.includes("?") ? "&" : "?";
+ const items = await fetchPage(`${url}${sep}per_page=${perPage}&page=${page}`, token);
+ out.push(...items.slice(0, want - out.length));
+ if (items.length < perPage) break;
+ }
+ return out;
+}
+
export async function fetchGitHubData(
repo: GitHubRepo,
token: string,
@@ -94,21 +119,23 @@ export async function fetchGitHubData(
const limit = opts.limit ?? 50;
const sinceParam = opts.since ? `&since=${encodeURIComponent(opts.since)}` : "";
- const issues = await ghFetch<GitHubIssue[]>(
- `${base}/issues?state=all&per_page=${limit}&sort=updated&direction=desc${sinceParam}`,
- token
+ const issues = await ghFetchList<GitHubIssue>(
+ `${base}/issues?state=all&sort=updated&direction=desc${sinceParam}`,
+ token,
+ limit,
);
const realIssues = issues.filter((i: any) => !i.pull_request);
- const pullRequests = await ghFetch<GitHubPR[]>(
- `${base}/pulls?state=all&per_page=${limit}&sort=updated&direction=desc`,
- token
+ const pullRequests = await ghFetchList<GitHubPR>(
+ `${base}/pulls?state=all&sort=updated&direction=desc`,
+ token,
+ limit,
);
- const commitLimit = Math.min(limit * 2, 100);
- const commits = await ghFetch<GitHubCommit[]>(
- `${base}/commits?per_page=${commitLimit}${sinceParam}`,
- token
+ const commits = await ghFetchList<GitHubCommit>(
+ `${base}/commits?${sinceParam.slice(1)}`.replace(/\?$/, ""),
+ token,
+ limit * 2,
);
const issueComments = new Map<number, GitHubComment[]>();