Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): don't lose deletions skipped by a failed hash lookup
KageBinary1dfbad72f04f
diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.tsindex 9330c097..8dca029e 100644--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts@@ -115,6 +115,8 @@ class FakeBackend {* path, which the tests written before it rely on.*/rememberHashes = false;+ /** Answer `/v1/source-hashes` with a 500, as a backend under load does. */+ failSourceHashes = false;private readonly hashes = new Map<string, { workspaceId: string | null; uri: string; hash: string }>();/** Forget every request so far, so a second run can be measured on its own. */@@ -305,6 +307,7 @@ class FakeBackend {if (path === "/v1/health") return send(200, { status: "ok", version: "1.0.28" });if (path === "/v1/source-hashes") {+ if (this.failSourceHashes) return send(500, { error: "500: transaction begin timeout" });if (!this.rememberHashes) return send(200, []);let uris: string[] = [];try { uris = (JSON.parse(body) as { uris?: string[] }).uris ?? []; } catch { /* none */ }@@ -935,6 +938,31 @@ describe("ingestFiles against a fake backend", () => {}});+ it("keeps a deletion it could not clean up in the baseline, so the next run retries it", async () => {+ // A failed hash lookup with deletions pending drops them for this run. The+ // baseline it then wrote came from the files still on disk, so it no+ // longer named the deleted file: no later run could find it deleted, and+ // its nodes stayed in the graph for good.+ fixture(3);+ backend.rememberHashes = true;+ const incremental = () =>+ ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });+ const baselineFiles = () =>+ Object.keys((JSON.parse(readFileSync(ingestMtimeCachePath(repo), "utf8")) as { files: Record<string, number> }).files);+ const deleted = join(repo, "src", "m001.ts");++ await incremental();+ expect(baselineFiles()).toContain(deleted);++ rmSync(deleted);+ execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });+ backend.failSourceHashes = true;+ await incremental();++ expect(baselineFiles(), "the skipped deletion is still pending").toContain(deleted);+ expect(baselineFiles()).toEqual(expect.arrayContaining([join(repo, "src", "m000.ts"), join(repo, "src", "m002.ts")]));+ });+it("re-sends commits that lost the base-rev race to another writer", async () => {// Two `ix map` runs against one backend: each commit that lands moves the// rev under the other's in-flight commit, which is answered 200diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 6fa746e4..73a78eba 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -912,6 +912,25 @@ const REBUILD_PROGRESS_SAVE_MS = 2_000;// Mtime cache — skip readFileSync+sha256 for unchanged files// ---------------------------------------------------------------------------+/**+ * The mtimes to record, plus the deleted files whose cleanup this run had to+ * skip, at their previous mtimes. Keeping them in the baseline is what makes+ * the next run find them deleted and retry.+ */+export function withDeferredDeletions(+ currentMtimes: Map<string, number>,+ previousMtimes: Map<string, number>,+ deferred: readonly string[],+): Map<string, number> {+ if (deferred.length === 0) return currentMtimes;+ const out = new Map(currentMtimes);+ for (const filePath of deferred) {+ const mtime = previousMtimes.get(filePath);+ if (mtime !== undefined && !out.has(filePath)) out.set(filePath, mtime);+ }+ return out;+}+export function persistIngestBaselineIfClean(projectRoot: string,mtimes: Map<string, number>,@@ -2129,8 +2148,12 @@ export async function ingestFiles(// fails closed when deletions are pending — but only the *deletions* need// that guarantee. Letting the throw escape aborted the entire map over one// transient blip, taking the ordinary ingest of every changed file with it.- // So catch it here: drop the deletions for this run (they stay in the- // baseline and are retried next time) and let the rest proceed.+ // So catch it here: drop the deletions for this run and let the rest+ // proceed. `deferredDeletions` carries them into the baseline this run+ // writes -- without it they vanished: `currentMtimes` never held a deleted+ // file, so the next run's baseline did not name them, they were never+ // found deleted again, and their nodes stayed in the graph for good.+ const deferredDeletions: string[] = [];if (hashLookupPaths.length > 0) {try {knownHashes = await loadExistingHashes(@@ -2143,6 +2166,7 @@ export async function ingestFiles();} catch (err) {if (debug) process.stderr.write(`\n [deletion cleanup skipped] hash lookup failed: ${err}\n`);+ deferredDeletions.push(...deletedPaths);deletedPaths.length = 0;knownHashes = new Map();}@@ -3436,7 +3460,7 @@ export async function ingestFiles(// missing from the graph until a --force.const baselinePersisted = persistIngestBaselineIfClean(projectRoot,- currentMtimes,+ withDeferredDeletions(currentMtimes, previousMtimes, deferredDeletions),latestRev,// Lost parses count as parse errors HERE, whatever they are called// elsewhere. A run whose worker pool died resolves every later file as