Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): resume an extractor re-ingest that was cut short
KageBinaryd3f4b5004640
diff --git a/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts b/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.tsindex b1ec0dd7..7dedc8e4 100644--- a/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-baseline-rev.test.ts@@ -5,8 +5,10 @@ import * as fs from "node:fs";import * as os from "node:os";import * as path from "node:path";-import { ingestMtimeCachePath } from "../config.js";-import { extractorChanged, loadIngestBaseline, saveIngestBaseline } from "../ingest-baseline.js";+import { clearIngestMtimeCache, ingestMtimeCachePath, ingestRebuildPath } from "../config.js";+import {+ clearRebuildProgress, extractorChanged, loadIngestBaseline, loadRebuildProgress, saveIngestBaseline, saveRebuildProgress,+} from "../ingest-baseline.js";import { advanceRev } from "../commands/ingest.js";let home: string;@@ -131,3 +133,35 @@ describe("ingest baseline extractor", () => {expect(extractorChanged(loadIngestBaseline(root), "tree-sitter/9.9")).toBe(true);});});++describe("extractor re-ingest progress", () => {+ const done = new Map<string, number>([["/p/a.ts", 1_000], ["/p/b.ts", 2_000]]);++ it("round-trips for the same root and extractor only", () => {+ const root = path.join(home, "p");+ saveRebuildProgress(root, "tree-sitter/9.9", done);+ expect(loadRebuildProgress(root, "tree-sitter/9.9")).toEqual(done);+ expect(loadRebuildProgress(root, "tree-sitter/9.10"), "progress toward another extractor").toBeNull();+ expect(loadRebuildProgress(path.join(home, "q"), "tree-sitter/9.9")).toBeNull();+ });++ it("reads a damaged file as no progress", () => {+ const root = path.join(home, "p");+ saveRebuildProgress(root, "tree-sitter/9.9", done);+ fs.writeFileSync(ingestRebuildPath(root), "{\"root\":");+ expect(loadRebuildProgress(root, "tree-sitter/9.9")).toBeNull();+ });++ it("is cleared on completion and with the mtime cache", () => {+ const root = path.join(home, "p");+ saveRebuildProgress(root, "tree-sitter/9.9", done);+ clearRebuildProgress(root);+ expect(fs.existsSync(ingestRebuildPath(root))).toBe(false);++ // `ix reset` and workspace migration clear the mtime cache: the progress+ // describes the same graph and must go with it.+ saveRebuildProgress(root, "tree-sitter/9.9", done);+ clearIngestMtimeCache(root);+ expect(fs.existsSync(ingestRebuildPath(root))).toBe(false);+ });+});diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.tsindex 0e725ee2..bd686a3f 100644--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts@@ -3,13 +3,13 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { createServer, type Server, type ServerResponse } from "node:http";import type { AddressInfo } from "node:net";-import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";+import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { execFileSync } from "node:child_process";import { ingestFiles } from "../commands/ingest.js";-import { ingestMtimeCachePath } from "../config.js";+import { ingestMtimeCachePath, ingestRebuildPath } from "../config.js";/*** Integration tests that drive `ingestFiles` end to end against a fake backend.@@ -102,6 +102,14 @@ class FakeBackend {bulk409AllLanded = false;/** Status for POST /v1/stitch. */stitchStatus = 200;+ /**+ * Answer `/v1/source-hashes` with the hashes of patches this fake accepted,+ * as the real backend does. Off by default: with it off every lookup is+ * empty, so the DB-reset guard sends each incremental run down the full+ * path, which the tests written before it rely on.+ */+ rememberHashes = false;+ private readonly hashes = new Map<string, { workspaceId: string | null; uri: string; hash: string }>();/** Forget every request so far, so a second run can be measured on its own. */resetRequests(): void {@@ -112,6 +120,13 @@ class FakeBackend {return this.requests.filter((r) => r.path === "/v1/stitch").length;}+ /** Patches in commit requests the backend ACCEPTED. */+ acceptedPatches(): number {+ return this.requests+ .filter(r => (r.path === "/v1/patches/bulk" || r.path === "/v1/patch") && r.code === 200)+ .reduce((sum, r) => sum + r.patches, 0);+ }+/** Bulk commits that the backend ACCEPTED, in order. */acceptedBulks(): number {return this.requests.filter(r => r.path === "/v1/patches/bulk" && r.code === 200).length;@@ -207,10 +222,11 @@ class FakeBackend {};if (path === "/v1/patches/bulk" || path === "/v1/patch") {- let patches: Array<{ patchId?: string }> = [];+ type SentPatch = { patchId?: string; source?: { uri?: string; sourceHash?: string; workspaceId?: string } };+ let patches: SentPatch[] = [];try {- const parsed = JSON.parse(body) as { patches?: Array<{ patchId?: string }> };- patches = parsed.patches ?? [parsed as { patchId?: string }];+ const parsed = JSON.parse(body) as { patches?: SentPatch[] };+ patches = parsed.patches ?? [parsed as SentPatch];} catch {/* a body we cannot read is still a request */}@@ -251,6 +267,14 @@ class FakeBackend {// and fires against a closed server.if (refused) return send(500, { error: "500: transaction begin timeout" });this.rev += patches.length || 1;+ if (this.rememberHashes) {+ for (const { source } of patches) {+ if (source?.uri && source.sourceHash) {+ this.hashes.set(`${source.workspaceId ?? ""}\0${source.uri}`,+ { workspaceId: source.workspaceId ?? null, uri: source.uri, hash: source.sourceHash });+ }+ }+ }// `status` included, because `PatchCommitResult` declares it required and// the ingest path branches on it. Serving 200s without it left// `result.status` undefined everywhere, so this fake could never produce@@ -270,7 +294,13 @@ class FakeBackend {}if (path === "/v1/health") return send(200, { status: "ok", version: "1.0.28" });- if (path === "/v1/source-hashes") return send(200, []);+ if (path === "/v1/source-hashes") {+ if (!this.rememberHashes) return send(200, []);+ let uris: string[] = [];+ try { uris = (JSON.parse(body) as { uris?: string[] }).uris ?? []; } catch { /* none */ }+ const wanted = new Set(uris);+ return send(200, [...this.hashes.values()].filter((row) => wanted.has(row.uri)));+ }if (path.startsWith("/v1/stitch/system/")) return send(200, { systemId: null });if (path === "/v1/stitch") {this.requests.push({ path, patches: 0 });@@ -858,6 +888,43 @@ describe("ingestFiles against a fake backend", () => {}});+ it("resumes an extractor re-ingest that was cut short instead of starting over", async () => {+ // The re-ingest an extractor change forces takes about twice a first map,+ // and an editor hook with a timeout can cut it short. The baseline is only+ // written by a run that finishes, so without recorded progress every later+ // run started over -- and could be cut short at the same point, forever.+ fixture(30);+ backend.rememberHashes = true;+ const incremental = () =>+ ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });+ const stored = () =>+ JSON.parse(readFileSync(ingestMtimeCachePath(repo), "utf8")) as { extractor?: string };+ const stderr = vi.spyOn(process.stderr, "write").mockImplementation(() => true);+ try {+ await incremental();+ const current = stored().extractor;+ writeFileSync(ingestMtimeCachePath(repo), JSON.stringify({ ...stored(), extractor: "tree-sitter/0.1" }));++ // Cut short: three files' patches never land, so the run cannot finish.+ backend.resetRequests();+ backend.poison = ["m007.ts", "m015.ts", "m023.ts"];+ await incremental().catch(() => undefined);+ expect(stored().extractor, "an unfinished re-ingest must not vouch for the new extractor").toBe("tree-sitter/0.1");+ expect(existsSync(ingestRebuildPath(repo)), "what landed is recorded").toBe(true);++ backend.resetRequests();+ backend.poison = [];+ await incremental();+ const notices = stderr.mock.calls.map(([chunk]) => String(chunk)).filter(c => c.includes("[extractor changed]"));+ expect(notices.at(-1)).toContain("Resuming the re-ingest: 27 of 30 files already done.");+ expect(backend.acceptedPatches(), "only the three that never landed are sent again").toBe(3);+ expect(stored().extractor).toBe(current);+ expect(existsSync(ingestRebuildPath(repo)), "a finished re-ingest leaves no progress behind").toBe(false);+ } finally {+ stderr.mockRestore();+ }+ });+it("commits a healthy repo in one bulk, with no per-file fan-out", async () => {fixture(12);diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 3ddfaddd..1f63ba92 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -13,7 +13,10 @@ import chalk from 'chalk';import { IxClient } from '../../client/api.js';import type { GraphPatchPayload } from '../../client/types.js';import { getEndpoint, resolveWorkspaceRoot, clearStitchScopeCache } from '../config.js';-import { extractorChanged, isRev, loadIngestBaseline, saveIngestBaseline } from '../ingest-baseline.js';+import {+ clearRebuildProgress, extractorChanged, isRev, loadIngestBaseline, loadRebuildProgress,+ saveIngestBaseline, saveRebuildProgress,+} from '../ingest-baseline.js';import { resolveGitHubToken } from '../github/auth.js';import { parseGitHubRepo, fetchGitHubData } from '../github/fetch.js';import { loadIngestionModules } from './ingestion-loader.js';@@ -870,6 +873,9 @@ export function advanceRev(current: number, incoming: unknown): number {return isRev(incoming) && incoming > current ? incoming : current;}+/** How often, at most, an extractor re-ingest records its progress to disk. */+const REBUILD_PROGRESS_SAVE_MS = 2_000;+// ---------------------------------------------------------------------------// Mtime cache — skip readFileSync+sha256 for unchanged files// ---------------------------------------------------------------------------@@ -1939,17 +1945,32 @@ export async function ingestFiles(// skip the mtime pre-filter and re-ingest everything, exactly like --force.const previousBaseline = loadIngestBaseline(projectRoot);const currentExtractor = extractorName();+ // A --lang or single-file run re-extracts only its own files, so it must+ // not record the new extractor for the rest; keep the old one and the next+ // full run still re-ingests them.+ const partialRun = Boolean(opts.lang) || stat.isFile();+ const baselineExtractor = partialRun ? previousBaseline?.extractor : currentExtractor;+ // Files a full re-ingest for this extractor has committed, recorded as they+ // land so a run cut short resumes instead of starting over; null when no+ // such re-ingest is under way.+ let rebuildProgress: Map<string, number> | null = null;+ let resumedRebuild: Map<string, number> | null = null;if (!opts.force && extractorChanged(previousBaseline, currentExtractor)) {+ if (!partialRun) {+ const done = workspaceMigrated ? null : loadRebuildProgress(projectRoot, currentExtractor);+ if (done && done.size > 0) resumedRebuild = done;+ rebuildProgress = new Map(done ?? []);+ }process.stderr.write(`\n [extractor changed] ${previousBaseline?.extractor ?? 'unrecorded'} -> ${currentExtractor}. ` +- `Re-ingesting every file once.\n`,+ (resumedRebuild+ ? `Resuming the re-ingest: ${resumedRebuild.size} of ${filePaths.length} files already done.\n`+ : `Re-ingesting every file once.\n`),);- opts.force = true;+ // A resumed re-ingest is not forced: the files it already committed are+ // skipped by mtime below, and every other file is forced one by one.+ if (!resumedRebuild) opts.force = true;}- // A --lang or single-file run re-extracts only its own files, so it must- // not record the new extractor for the rest; keep the old one and the next- // full run still re-ingests them.- const baselineExtractor = (opts.lang || stat.isFile()) ? previousBaseline?.extractor : currentExtractor;const previousMtimes = previousBaseline?.files ?? new Map<string, number>();const {previousDeletedFiles,@@ -1962,7 +1983,28 @@ export async function ingestFiles();const mtimeCache = (opts.force || workspaceMigrated)? new Map<string, number>()- : new Map(previousMtimes);+ : new Map(resumedRebuild ?? previousMtimes);+ if (resumedRebuild) {+ for (const filePath of filePaths) if (!resumedRebuild.has(filePath)) forceReingestPaths.add(filePath);+ }+ let rebuildSavedAt = 0;+ // Committed patches carry the workspace-relative path; progress is keyed+ // like the mtime baseline, by absolute path. Built on first use, after the+ // stat loop below has filled `currentMtimes`.+ let absoluteOf: Map<string, string> | undefined;+ const noteRebuilt = rebuildProgress === null ? undefined : (item: { filePath: string }): void => {+ absoluteOf ??= new Map([...currentMtimes.keys()].map(abs => [toWorkspaceRelative(abs), abs]));+ const abs = absoluteOf.get(item.filePath);+ const mtime = abs === undefined ? undefined : currentMtimes.get(abs);+ if (abs === undefined || mtime === undefined) return;+ rebuildProgress!.set(abs, mtime);+ // Throttled: the whole map is rewritten each time, and a cut-short run+ // loses at most this much of its work.+ if (Date.now() - rebuildSavedAt >= REBUILD_PROGRESS_SAVE_MS) {+ rebuildSavedAt = Date.now();+ saveRebuildProgress(projectRoot, currentExtractor, rebuildProgress!);+ }+ };const currentMtimes = new Map<string, number>();// DB-reset guard: if the mtime cache has entries but the server returns no hashes@@ -1973,6 +2015,8 @@ export async function ingestFiles(const sampleHashes = await loadExistingHashes(client, samplePaths, toWorkspaceRelative, sourceWorkspaceIdOf, debug);if (sampleHashes.size === 0) {mtimeCache.clear();+ // What an unfinished re-ingest committed is gone with the rest.+ rebuildProgress?.clear();if (debug) process.stderr.write(`\n DB reset detected — invalidating mtime cache\n`);}}@@ -2846,7 +2890,7 @@ export async function ingestFiles(}}if (preparedPatches.length === 0) return;- commitMs += await commitPreparedPatches(preparedPatches, batch.length);+ commitMs += await commitPreparedPatches(preparedPatches, batch.length, { onCommitted: noteRebuilt });} finally {const totalSaveMs = Math.round(performance.now() - saveStart);timings.resolveMs += resolveEdgesMs;@@ -2927,7 +2971,7 @@ export async function ingestFiles(progressCurrent = allParsed.length;return;}- commitMs += await commitPreparedPatches(preparedPatches, allParsed.length, { updateProgress: true });+ commitMs += await commitPreparedPatches(preparedPatches, allParsed.length, { updateProgress: true, onCommitted: noteRebuilt });} finally {const totalSaveMs = Math.round(performance.now() - saveStart);timings.resolveMs += resolveEdgesMs;@@ -3311,7 +3355,7 @@ export async function ingestFiles(// three guards would start caching mtimes for files whose patches never// landed. The next run would skip them as unchanged and they would stay// missing from the graph until a --force.- persistIngestBaselineIfClean(+ const baselinePersisted = persistIngestBaselineIfClean(projectRoot,currentMtimes,latestRev,@@ -3329,6 +3373,12 @@ export async function ingestFiles(nextDeletedFiles,baselineExtractor,);+ if (rebuildProgress !== null) {+ // Finished: the baseline now records the new extractor. Otherwise keep+ // what landed, so the next run resumes from here.+ if (baselinePersisted) clearRebuildProgress(projectRoot);+ else saveRebuildProgress(projectRoot, currentExtractor, rebuildProgress);+ }// Migration cleanup (Ix#225 gap 2): the re-ingest under the new path-based id has// committed, so delete the OLD id's now-orphaned nodes/edges/patches. Best-effort —diff --git a/ix-cli/src/cli/config.ts b/ix-cli/src/cli/config.tsindex 178826f3..e7d84d67 100644--- a/ix-cli/src/cli/config.ts+++ b/ix-cli/src/cli/config.ts@@ -18,6 +18,16 @@ export function ingestMtimeCachePath(projectRoot: string): string {return join(ixHome(), `ingest_mtimes_${key}.json`);}+/**+ * Path to the progress of an unfinished re-ingest for a new extractor (see+ * `loadRebuildProgress`). Cleared with the mtime cache, since it describes the+ * same graph.+ */+export function ingestRebuildPath(projectRoot: string): string {+ const key = createHash("sha256").update(projectRoot).digest("hex").slice(0, 12);+ return join(ixHome(), `ingest_rebuild_${key}.json`);+}+/** Path to the architecture-map completion marker for one project root. */export function mapBaselinePath(projectRoot: string): string {const key = createHash("sha256").update(projectRoot).digest("hex").slice(0, 12);@@ -89,6 +99,7 @@ export function clearMapBaseline(projectRoot: string): void {*/export function clearIngestMtimeCache(projectRoot: string): void {try { rmSync(ingestMtimeCachePath(projectRoot), { force: true }); } catch { /* non-critical */ }+ try { rmSync(ingestRebuildPath(projectRoot), { force: true }); } catch { /* non-critical */ }clearMapBaseline(projectRoot);}diff --git a/ix-cli/src/cli/ingest-baseline.ts b/ix-cli/src/cli/ingest-baseline.tsindex 6678571b..1ebd529d 100644--- a/ix-cli/src/cli/ingest-baseline.ts+++ b/ix-cli/src/cli/ingest-baseline.ts@@ -2,7 +2,7 @@import * as fs from "node:fs";import * as path from "node:path";-import { ingestMtimeCachePath } from "./config.js";+import { ingestMtimeCachePath, ingestRebuildPath } from "./config.js";interface SerializedIngestBaseline {root: string;@@ -139,3 +139,49 @@ export function saveIngestBaseline(export function extractorChanged(baseline: IngestBaseline | null, current: string): boolean {return baseline !== null && baseline.extractor !== current;}++/**+ * Files an unfinished re-ingest for `extractor` has already committed, with the+ * mtime each had then; null when there is none for this extractor.+ *+ * The re-ingest an extractor change forces takes about twice a first map, and+ * the mtime baseline is written only when a run finishes. So a run cut short --+ * an editor hook's timeout, a closed terminal -- used to leave nothing behind,+ * and every later run started over and could be cut short at the same point.+ * Recording progress as files land lets the next run skip what is done.+ */+export function loadRebuildProgress(projectRoot: string, extractor: string): Map<string, number> | null {+ try {+ const data = JSON.parse(fs.readFileSync(ingestRebuildPath(projectRoot), "utf-8")) as {+ root?: unknown; extractor?: unknown; files?: Record<string, unknown>;+ };+ if (data.root !== projectRoot || data.extractor !== extractor || !data.files || typeof data.files !== "object") {+ return null;+ }+ const files = new Map<string, number>();+ for (const [filePath, mtime] of Object.entries(data.files)) {+ if (typeof mtime === "number" && Number.isFinite(mtime)) files.set(filePath, mtime);+ }+ return files;+ } catch {+ return null;+ }+}++export function saveRebuildProgress(projectRoot: string, extractor: string, files: Map<string, number>): void {+ try {+ const target = ingestRebuildPath(projectRoot);+ fs.mkdirSync(path.dirname(target), { recursive: true });+ // Via a temp file and rename: a process killed mid-write would otherwise+ // leave a truncated file, and the next run would start over.+ const tmp = `${target}.${process.pid}.tmp`;+ fs.writeFileSync(tmp, JSON.stringify({ root: projectRoot, extractor, files: Object.fromEntries(files) }));+ fs.renameSync(tmp, target);+ } catch {+ // Progress is an optimization; losing it costs a longer re-ingest, not data.+ }+}++export function clearRebuildProgress(projectRoot: string): void {+ try { fs.rmSync(ingestRebuildPath(projectRoot), { force: true }); } catch { /* non-critical */ }+}