Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): resume an extractor re-ingest that was cut short

KageBinaryd3f4b5004640
diff --git a/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts b/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts
index b1ec0dd7..7dedc8e4 100644
--- a/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts
@@ -5,8 +5,10 @@ import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
-import { ingestMtimeCachePath } from "../config.js";
-import { extractorChanged, loadIngestBaseline, saveIngestBaseline } from "../ingest-baseline.js";
+import { clearIngestMtimeCache, ingestMtimeCachePath, ingestRebuildPath } from "../config.js";
+import {
+ clearRebuildProgress, extractorChanged, loadIngestBaseline, loadRebuildProgress, saveIngestBaseline, saveRebuildProgress,
+} from "../ingest-baseline.js";
import { advanceRev } from "../commands/ingest.js";
let home: string;
@@ -131,3 +133,35 @@ describe("ingest baseline extractor", () => {
expect(extractorChanged(loadIngestBaseline(root), "tree-sitter/9.9")).toBe(true);
});
});
+
+describe("extractor re-ingest progress", () => {
+ const done = new Map<string, number>([["/p/a.ts", 1_000], ["/p/b.ts", 2_000]]);
+
+ it("round-trips for the same root and extractor only", () => {
+ const root = path.join(home, "p");
+ saveRebuildProgress(root, "tree-sitter/9.9", done);
+ expect(loadRebuildProgress(root, "tree-sitter/9.9")).toEqual(done);
+ expect(loadRebuildProgress(root, "tree-sitter/9.10"), "progress toward another extractor").toBeNull();
+ expect(loadRebuildProgress(path.join(home, "q"), "tree-sitter/9.9")).toBeNull();
+ });
+
+ it("reads a damaged file as no progress", () => {
+ const root = path.join(home, "p");
+ saveRebuildProgress(root, "tree-sitter/9.9", done);
+ fs.writeFileSync(ingestRebuildPath(root), "{\"root\":");
+ expect(loadRebuildProgress(root, "tree-sitter/9.9")).toBeNull();
+ });
+
+ it("is cleared on completion and with the mtime cache", () => {
+ const root = path.join(home, "p");
+ saveRebuildProgress(root, "tree-sitter/9.9", done);
+ clearRebuildProgress(root);
+ expect(fs.existsSync(ingestRebuildPath(root))).toBe(false);
+
+ // `ix reset` and workspace migration clear the mtime cache: the progress
+ // describes the same graph and must go with it.
+ saveRebuildProgress(root, "tree-sitter/9.9", done);
+ clearIngestMtimeCache(root);
+ expect(fs.existsSync(ingestRebuildPath(root))).toBe(false);
+ });
+});
diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.ts
index 0e725ee2..bd686a3f 100644
--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts
@@ -3,13 +3,13 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createServer, type Server, type ServerResponse } from "node:http";
import type { AddressInfo } from "node:net";
-import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { execFileSync } from "node:child_process";
import { ingestFiles } from "../commands/ingest.js";
-import { ingestMtimeCachePath } from "../config.js";
+import { ingestMtimeCachePath, ingestRebuildPath } from "../config.js";
/**
* Integration tests that drive `ingestFiles` end to end against a fake backend.
@@ -102,6 +102,14 @@ class FakeBackend {
bulk409AllLanded = false;
/** Status for POST /v1/stitch. */
stitchStatus = 200;
+ /**
+ * Answer `/v1/source-hashes` with the hashes of patches this fake accepted,
+ * as the real backend does. Off by default: with it off every lookup is
+ * empty, so the DB-reset guard sends each incremental run down the full
+ * path, which the tests written before it rely on.
+ */
+ rememberHashes = false;
+ private readonly hashes = new Map<string, { workspaceId: string | null; uri: string; hash: string }>();
/** Forget every request so far, so a second run can be measured on its own. */
resetRequests(): void {
@@ -112,6 +120,13 @@ class FakeBackend {
return this.requests.filter((r) => r.path === "/v1/stitch").length;
}
+ /** Patches in commit requests the backend ACCEPTED. */
+ acceptedPatches(): number {
+ return this.requests
+ .filter(r => (r.path === "/v1/patches/bulk" || r.path === "/v1/patch") && r.code === 200)
+ .reduce((sum, r) => sum + r.patches, 0);
+ }
+
/** Bulk commits that the backend ACCEPTED, in order. */
acceptedBulks(): number {
return this.requests.filter(r => r.path === "/v1/patches/bulk" && r.code === 200).length;
@@ -207,10 +222,11 @@ class FakeBackend {
};
if (path === "/v1/patches/bulk" || path === "/v1/patch") {
- let patches: Array<{ patchId?: string }> = [];
+ type SentPatch = { patchId?: string; source?: { uri?: string; sourceHash?: string; workspaceId?: string } };
+ let patches: SentPatch[] = [];
try {
- const parsed = JSON.parse(body) as { patches?: Array<{ patchId?: string }> };
- patches = parsed.patches ?? [parsed as { patchId?: string }];
+ const parsed = JSON.parse(body) as { patches?: SentPatch[] };
+ patches = parsed.patches ?? [parsed as SentPatch];
} catch {
/* a body we cannot read is still a request */
}
@@ -251,6 +267,14 @@ class FakeBackend {
// and fires against a closed server.
if (refused) return send(500, { error: "500: transaction begin timeout" });
this.rev += patches.length || 1;
+ if (this.rememberHashes) {
+ for (const { source } of patches) {
+ if (source?.uri && source.sourceHash) {
+ this.hashes.set(`${source.workspaceId ?? ""}\0${source.uri}`,
+ { workspaceId: source.workspaceId ?? null, uri: source.uri, hash: source.sourceHash });
+ }
+ }
+ }
// `status` included, because `PatchCommitResult` declares it required and
// the ingest path branches on it. Serving 200s without it left
// `result.status` undefined everywhere, so this fake could never produce
@@ -270,7 +294,13 @@ class FakeBackend {
}
if (path === "/v1/health") return send(200, { status: "ok", version: "1.0.28" });
- if (path === "/v1/source-hashes") return send(200, []);
+ if (path === "/v1/source-hashes") {
+ if (!this.rememberHashes) return send(200, []);
+ let uris: string[] = [];
+ try { uris = (JSON.parse(body) as { uris?: string[] }).uris ?? []; } catch { /* none */ }
+ const wanted = new Set(uris);
+ return send(200, [...this.hashes.values()].filter((row) => wanted.has(row.uri)));
+ }
if (path.startsWith("/v1/stitch/system/")) return send(200, { systemId: null });
if (path === "/v1/stitch") {
this.requests.push({ path, patches: 0 });
@@ -858,6 +888,43 @@ describe("ingestFiles against a fake backend", () => {
}
});
+ it("resumes an extractor re-ingest that was cut short instead of starting over", async () => {
+ // The re-ingest an extractor change forces takes about twice a first map,
+ // and an editor hook with a timeout can cut it short. The baseline is only
+ // written by a run that finishes, so without recorded progress every later
+ // run started over -- and could be cut short at the same point, forever.
+ fixture(30);
+ backend.rememberHashes = true;
+ const incremental = () =>
+ ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });
+ const stored = () =>
+ JSON.parse(readFileSync(ingestMtimeCachePath(repo), "utf8")) as { extractor?: string };
+ const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);
+ try {
+ await incremental();
+ const current = stored().extractor;
+ writeFileSync(ingestMtimeCachePath(repo), JSON.stringify({ ...stored(), extractor: "tree-sitter/0.1" }));
+
+ // Cut short: three files' patches never land, so the run cannot finish.
+ backend.resetRequests();
+ backend.poison = ["m007.ts", "m015.ts", "m023.ts"];
+ await incremental().catch(() => undefined);
+ expect(stored().extractor, "an unfinished re-ingest must not vouch for the new extractor").toBe("tree-sitter/0.1");
+ expect(existsSync(ingestRebuildPath(repo)), "what landed is recorded").toBe(true);
+
+ backend.resetRequests();
+ backend.poison = [];
+ await incremental();
+ const notices = stderr.mock.calls.map(([chunk]) => String(chunk)).filter(c => c.includes("[extractor changed]"));
+ expect(notices.at(-1)).toContain("Resuming the re-ingest: 27 of 30 files already done.");
+ expect(backend.acceptedPatches(), "only the three that never landed are sent again").toBe(3);
+ expect(stored().extractor).toBe(current);
+ expect(existsSync(ingestRebuildPath(repo)), "a finished re-ingest leaves no progress behind").toBe(false);
+ } finally {
+ stderr.mockRestore();
+ }
+ });
+
it("commits a healthy repo in one bulk, with no per-file fan-out", async () => {
fixture(12);
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 3ddfaddd..1f63ba92 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -13,7 +13,10 @@ import chalk from 'chalk';
import { IxClient } from '../../client/api.js';
import type { GraphPatchPayload } from '../../client/types.js';
import { getEndpoint, resolveWorkspaceRoot, clearStitchScopeCache } from '../config.js';
-import { extractorChanged, isRev, loadIngestBaseline, saveIngestBaseline } from '../ingest-baseline.js';
+import {
+ clearRebuildProgress, extractorChanged, isRev, loadIngestBaseline, loadRebuildProgress,
+ saveIngestBaseline, saveRebuildProgress,
+} from '../ingest-baseline.js';
import { resolveGitHubToken } from '../github/auth.js';
import { parseGitHubRepo, fetchGitHubData } from '../github/fetch.js';
import { loadIngestionModules } from './ingestion-loader.js';
@@ -870,6 +873,9 @@ export function advanceRev(current: number, incoming: unknown): number {
return isRev(incoming) && incoming > current ? incoming : current;
}
+/** How often, at most, an extractor re-ingest records its progress to disk. */
+const REBUILD_PROGRESS_SAVE_MS = 2_000;
+
// ---------------------------------------------------------------------------
// Mtime cache — skip readFileSync+sha256 for unchanged files
// ---------------------------------------------------------------------------
@@ -1939,17 +1945,32 @@ export async function ingestFiles(
// skip the mtime pre-filter and re-ingest everything, exactly like --force.
const previousBaseline = loadIngestBaseline(projectRoot);
const currentExtractor = extractorName();
+ // A --lang or single-file run re-extracts only its own files, so it must
+ // not record the new extractor for the rest; keep the old one and the next
+ // full run still re-ingests them.
+ const partialRun = Boolean(opts.lang) || stat.isFile();
+ const baselineExtractor = partialRun ? previousBaseline?.extractor : currentExtractor;
+ // Files a full re-ingest for this extractor has committed, recorded as they
+ // land so a run cut short resumes instead of starting over; null when no
+ // such re-ingest is under way.
+ let rebuildProgress: Map<string, number> | null = null;
+ let resumedRebuild: Map<string, number> | null = null;
if (!opts.force && extractorChanged(previousBaseline, currentExtractor)) {
+ if (!partialRun) {
+ const done = workspaceMigrated ? null : loadRebuildProgress(projectRoot, currentExtractor);
+ if (done && done.size > 0) resumedRebuild = done;
+ rebuildProgress = new Map(done ?? []);
+ }
process.stderr.write(
`\n [extractor changed] ${previousBaseline?.extractor ?? 'unrecorded'} -> ${currentExtractor}. ` +
- `Re-ingesting every file once.\n`,
+ (resumedRebuild
+ ? `Resuming the re-ingest: ${resumedRebuild.size} of ${filePaths.length} files already done.\n`
+ : `Re-ingesting every file once.\n`),
);
- opts.force = true;
+ // A resumed re-ingest is not forced: the files it already committed are
+ // skipped by mtime below, and every other file is forced one by one.
+ if (!resumedRebuild) opts.force = true;
}
- // A --lang or single-file run re-extracts only its own files, so it must
- // not record the new extractor for the rest; keep the old one and the next
- // full run still re-ingests them.
- const baselineExtractor = (opts.lang || stat.isFile()) ? previousBaseline?.extractor : currentExtractor;
const previousMtimes = previousBaseline?.files ?? new Map<string, number>();
const {
previousDeletedFiles,
@@ -1962,7 +1983,28 @@ export async function ingestFiles(
);
const mtimeCache = (opts.force || workspaceMigrated)
? new Map<string, number>()
- : new Map(previousMtimes);
+ : new Map(resumedRebuild ?? previousMtimes);
+ if (resumedRebuild) {
+ for (const filePath of filePaths) if (!resumedRebuild.has(filePath)) forceReingestPaths.add(filePath);
+ }
+ let rebuildSavedAt = 0;
+ // Committed patches carry the workspace-relative path; progress is keyed
+ // like the mtime baseline, by absolute path. Built on first use, after the
+ // stat loop below has filled `currentMtimes`.
+ let absoluteOf: Map<string, string> | undefined;
+ const noteRebuilt = rebuildProgress === null ? undefined : (item: { filePath: string }): void => {
+ absoluteOf ??= new Map([...currentMtimes.keys()].map(abs => [toWorkspaceRelative(abs), abs]));
+ const abs = absoluteOf.get(item.filePath);
+ const mtime = abs === undefined ? undefined : currentMtimes.get(abs);
+ if (abs === undefined || mtime === undefined) return;
+ rebuildProgress!.set(abs, mtime);
+ // Throttled: the whole map is rewritten each time, and a cut-short run
+ // loses at most this much of its work.
+ if (Date.now() - rebuildSavedAt >= REBUILD_PROGRESS_SAVE_MS) {
+ rebuildSavedAt = Date.now();
+ saveRebuildProgress(projectRoot, currentExtractor, rebuildProgress!);
+ }
+ };
const currentMtimes = new Map<string, number>();
// DB-reset guard: if the mtime cache has entries but the server returns no hashes
@@ -1973,6 +2015,8 @@ export async function ingestFiles(
const sampleHashes = await loadExistingHashes(client, samplePaths, toWorkspaceRelative, sourceWorkspaceIdOf, debug);
if (sampleHashes.size === 0) {
mtimeCache.clear();
+ // What an unfinished re-ingest committed is gone with the rest.
+ rebuildProgress?.clear();
if (debug) process.stderr.write(`\n DB reset detected — invalidating mtime cache\n`);
}
}
@@ -2846,7 +2890,7 @@ export async function ingestFiles(
}
}
if (preparedPatches.length === 0) return;
- commitMs += await commitPreparedPatches(preparedPatches, batch.length);
+ commitMs += await commitPreparedPatches(preparedPatches, batch.length, { onCommitted: noteRebuilt });
} finally {
const totalSaveMs = Math.round(performance.now() - saveStart);
timings.resolveMs += resolveEdgesMs;
@@ -2927,7 +2971,7 @@ export async function ingestFiles(
progressCurrent = allParsed.length;
return;
}
- commitMs += await commitPreparedPatches(preparedPatches, allParsed.length, { updateProgress: true });
+ commitMs += await commitPreparedPatches(preparedPatches, allParsed.length, { updateProgress: true, onCommitted: noteRebuilt });
} finally {
const totalSaveMs = Math.round(performance.now() - saveStart);
timings.resolveMs += resolveEdgesMs;
@@ -3311,7 +3355,7 @@ export async function ingestFiles(
// three guards would start caching mtimes for files whose patches never
// landed. The next run would skip them as unchanged and they would stay
// missing from the graph until a --force.
- persistIngestBaselineIfClean(
+ const baselinePersisted = persistIngestBaselineIfClean(
projectRoot,
currentMtimes,
latestRev,
@@ -3329,6 +3373,12 @@ export async function ingestFiles(
nextDeletedFiles,
baselineExtractor,
);
+ if (rebuildProgress !== null) {
+ // Finished: the baseline now records the new extractor. Otherwise keep
+ // what landed, so the next run resumes from here.
+ if (baselinePersisted) clearRebuildProgress(projectRoot);
+ else saveRebuildProgress(projectRoot, currentExtractor, rebuildProgress);
+ }
// Migration cleanup (Ix#225 gap 2): the re-ingest under the new path-based id has
// committed, so delete the OLD id's now-orphaned nodes/edges/patches. Best-effort —
diff --git a/ix-cli/src/cli/config.ts b/ix-cli/src/cli/config.ts
index 178826f3..e7d84d67 100644
--- a/ix-cli/src/cli/config.ts
+++ b/ix-cli/src/cli/config.ts
@@ -18,6 +18,16 @@ export function ingestMtimeCachePath(projectRoot: string): string {
return join(ixHome(), `ingest_mtimes_${key}.json`);
}
+/**
+ * Path to the progress of an unfinished re-ingest for a new extractor (see
+ * `loadRebuildProgress`). Cleared with the mtime cache, since it describes the
+ * same graph.
+ */
+export function ingestRebuildPath(projectRoot: string): string {
+ const key = createHash("sha256").update(projectRoot).digest("hex").slice(0, 12);
+ return join(ixHome(), `ingest_rebuild_${key}.json`);
+}
+
/** Path to the architecture-map completion marker for one project root. */
export function mapBaselinePath(projectRoot: string): string {
const key = createHash("sha256").update(projectRoot).digest("hex").slice(0, 12);
@@ -89,6 +99,7 @@ export function clearMapBaseline(projectRoot: string): void {
*/
export function clearIngestMtimeCache(projectRoot: string): void {
try { rmSync(ingestMtimeCachePath(projectRoot), { force: true }); } catch { /* non-critical */ }
+ try { rmSync(ingestRebuildPath(projectRoot), { force: true }); } catch { /* non-critical */ }
clearMapBaseline(projectRoot);
}
diff --git a/ix-cli/src/cli/ingest-baseline.ts b/ix-cli/src/cli/ingest-baseline.ts
index 6678571b..1ebd529d 100644
--- a/ix-cli/src/cli/ingest-baseline.ts
+++ b/ix-cli/src/cli/ingest-baseline.ts
@@ -2,7 +2,7 @@
import * as fs from "node:fs";
import * as path from "node:path";
-import { ingestMtimeCachePath } from "./config.js";
+import { ingestMtimeCachePath, ingestRebuildPath } from "./config.js";
interface SerializedIngestBaseline {
root: string;
@@ -139,3 +139,49 @@ export function saveIngestBaseline(
export function extractorChanged(baseline: IngestBaseline | null, current: string): boolean {
return baseline !== null && baseline.extractor !== current;
}
+
+/**
+ * Files an unfinished re-ingest for `extractor` has already committed, with the
+ * mtime each had then; null when there is none for this extractor.
+ *
+ * The re-ingest an extractor change forces takes about twice a first map, and
+ * the mtime baseline is written only when a run finishes. So a run cut short --
+ * an editor hook's timeout, a closed terminal -- used to leave nothing behind,
+ * and every later run started over and could be cut short at the same point.
+ * Recording progress as files land lets the next run skip what is done.
+ */
+export function loadRebuildProgress(projectRoot: string, extractor: string): Map<string, number> | null {
+ try {
+ const data = JSON.parse(fs.readFileSync(ingestRebuildPath(projectRoot), "utf-8")) as {
+ root?: unknown; extractor?: unknown; files?: Record<string, unknown>;
+ };
+ if (data.root !== projectRoot || data.extractor !== extractor || !data.files || typeof data.files !== "object") {
+ return null;
+ }
+ const files = new Map<string, number>();
+ for (const [filePath, mtime] of Object.entries(data.files)) {
+ if (typeof mtime === "number" && Number.isFinite(mtime)) files.set(filePath, mtime);
+ }
+ return files;
+ } catch {
+ return null;
+ }
+}
+
+export function saveRebuildProgress(projectRoot: string, extractor: string, files: Map<string, number>): void {
+ try {
+ const target = ingestRebuildPath(projectRoot);
+ fs.mkdirSync(path.dirname(target), { recursive: true });
+ // Via a temp file and rename: a process killed mid-write would otherwise
+ // leave a truncated file, and the next run would start over.
+ const tmp = `${target}.${process.pid}.tmp`;
+ fs.writeFileSync(tmp, JSON.stringify({ root: projectRoot, extractor, files: Object.fromEntries(files) }));
+ fs.renameSync(tmp, target);
+ } catch {
+ // Progress is an optimization; losing it costs a longer re-ingest, not data.
+ }
+}
+
+export function clearRebuildProgress(projectRoot: string): void {
+ try { fs.rmSync(ingestRebuildPath(projectRoot), { force: true }); } catch { /* non-critical */ }
+}