Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): remove stale graph entities (#378)
Hiro-Chibab25bf2efe837
diff --git a/core-ingestion/src/__tests__/patchBuilder.test.ts b/core-ingestion/src/__tests__/patchBuilder.test.tsindex 03a533b7..745c450c 100644--- a/core-ingestion/src/__tests__/patchBuilder.test.ts+++ b/core-ingestion/src/__tests__/patchBuilder.test.ts@@ -1,6 +1,11 @@import { describe, expect, it } from 'vitest';-import { extractorName, PREVIOUS_EXTRACTORS } from '../patch-builder.js';+import {+ buildDeletionPatch,+ extractorName,+ PREVIOUS_EXTRACTORS,+ sourcePatchIdCandidates,+} from '../patch-builder.js';describe('patch-builder extractor policy', () => {it('tracks the immediate predecessor when the extractor version is bumped', () => {@@ -17,4 +22,29 @@ describe('patch-builder extractor policy', () => {expect(PREVIOUS_EXTRACTORS).not.toContain(current);expect(PREVIOUS_EXTRACTORS.every(version => /^tree-sitter\/\d+\.\d+$/.test(version))).toBe(true);});++ it('builds a deterministic tombstone for every supported prior patch id', () => {+ const filePath = 'src/removed.ts';+ const sourceHash = 'previous-source-hash';+ const workspaceId = 'deadbeef';+ const candidates = sourcePatchIdCandidates(filePath, sourceHash, workspaceId);+ const ops = [{ type: 'DeleteNode', id: '00000000-0000-0000-0000-000000000001' }];++ const patch = buildDeletionPatch(filePath, sourceHash, 'baseline-1', workspaceId, ops);+ const retry = buildDeletionPatch(filePath, sourceHash, 'baseline-1', workspaceId, ops);+ const nextDeletion = buildDeletionPatch(filePath, sourceHash, 'baseline-2', workspaceId, ops);++ expect(new Set(candidates).size).toBe(PREVIOUS_EXTRACTORS.length + 2);+ expect(patch.patchId).toBe(retry.patchId);+ expect(patch.patchId).not.toBe(nextDeletion.patchId);+ expect(patch.source).toMatchObject({+ uri: filePath,+ extractor: extractorName(),+ sourceType: 'code',+ workspaceId,+ });+ expect(patch.source.sourceHash).not.toBe(sourceHash);+ expect(patch.ops).toEqual(ops);+ expect(patch.replaces).toEqual(candidates);+ });});diff --git a/core-ingestion/src/patch-builder.ts b/core-ingestion/src/patch-builder.tsindex 8e381f21..ee2e9d2d 100644--- a/core-ingestion/src/patch-builder.ts+++ b/core-ingestion/src/patch-builder.ts@@ -95,6 +95,20 @@ export function extractorName(): string {/** Previous extractor versions — their patches are superseded when re-ingesting. */export const PREVIOUS_EXTRACTORS = ['tree-sitter/1.23', 'tree-sitter/1.22', 'tree-sitter/1.21', 'tree-sitter/1.20', 'tree-sitter/1.19', 'tree-sitter/1.18', 'tree-sitter/1.17', 'tree-sitter/1.16', 'tree-sitter/1.15', 'tree-sitter/1.14', 'tree-sitter/1.13', 'tree-sitter/1.12', 'tree-sitter/1.11', 'tree-sitter/1.10', 'tree-sitter/1.9', 'tree-sitter/1.8', 'tree-sitter/1.7', 'tree-sitter/1.6', 'tree-sitter/1.5', 'tree-sitter/1.4', 'tree-sitter/1.3', 'tree-sitter/1.2', 'tree-sitter/1.1'];+/** Patch ids that may own the active graph entities for a stored source hash. */+export function sourcePatchIdCandidates(+ filePath: string,+ sourceHash: string,+ workspaceId: string,+): string[] {+ const { computePatchId, legacyPatchId } = makeIds(workspaceId);+ return [+ computePatchId(filePath, sourceHash, extractorName()),+ ...PREVIOUS_EXTRACTORS.map(extractor => computePatchId(filePath, sourceHash, extractor)),+ legacyPatchId(filePath, sourceHash),+ ];+}+// nodeId / edgeId / chunkId / computePatchId / legacyPatchId are produced by// makeIds(workspaceId) above so every id is scoped to its workspace.@@ -379,6 +393,42 @@ export interface MultiRepoContext {repoWorkspaceOf?: (filePath: string) => string | undefined;}+/** Build a patch that retires every entity previously emitted for a deleted file. */+export function buildDeletionPatch(+ filePath: string,+ previousSourceHash: string,+ deletionToken: string,+ workspaceId: string,+ ops: PatchOp[],+ multiRepo?: MultiRepoContext,+): GraphPatchPayload {+ const extractor = extractorName();+ const deletionHash = crypto+ .createHash('sha256')+ .update(`deleted:${previousSourceHash}:${deletionToken}`)+ .digest('hex');+ const { computePatchId } = makeIds(workspaceId);++ return {+ patchId: computePatchId(filePath, deletionHash, extractor),+ actor: 'ix/ingestion',+ timestamp: new Date().toISOString(),+ source: {+ uri: filePath,+ sourceHash: deletionHash,+ extractor,+ sourceType: sourceType(filePath),+ workspaceId,+ ...(multiRepo?.systemId ? { systemId: multiRepo.systemId } : {}),+ ...(multiRepo?.repoId ? { repoId: multiRepo.repoId } : {}),+ },+ baseRev: 0,+ ops,+ replaces: sourcePatchIdCandidates(filePath, previousSourceHash, workspaceId),+ intent: `Deleted ${nodePath.basename(filePath)}`,+ };+}+export function buildPatchWithResolution(result: FileParseResult,sourceHash: string,diff --git a/ix-cli/src/cli/__tests__/ingest-reconcile.test.ts b/ix-cli/src/cli/__tests__/ingest-reconcile.test.tsnew file mode 100644index 00000000..54fddc27--- /dev/null+++ b/ix-cli/src/cli/__tests__/ingest-reconcile.test.ts@@ -0,0 +1,216 @@+import { describe, expect, it, vi } from "vitest";+import * as nodePath from "node:path";++import type { GraphPatchPayload } from "../../client/types.js";+import {+ patchRequiresPerFileCommit,+ planDeletedFileRecovery,+ reconcileRemovedEntities,+} from "../commands/ingest.js";++function patchWith(ops: GraphPatchPayload["ops"]): GraphPatchPayload {+ return {+ patchId: "next-patch",+ actor: "ix/ingestion",+ timestamp: "2026-08-10T00:00:00.000Z",+ source: {+ uri: "src/example.ts",+ sourceHash: "next-hash",+ extractor: "tree-sitter/1.24",+ sourceType: "code",+ workspaceId: "deadbeef",+ },+ baseRev: 0,+ ops,+ replaces: [],+ };+}++describe("reconcileRemovedEntities", () => {+ it("routes patches with deletion ops away from the bulk endpoint", () => {+ expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteNode", id: "old-node" }]))).toBe(true);+ expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteEdge", id: "old-edge" }]))).toBe(true);+ expect(patchRequiresPerFileCommit(patchWith([{ type: "UpsertNode", id: "node" }]))).toBe(false);+ });++ it("reingests surviving dependents when a deleted file returns", () => {+ const projectRoot = nodePath.resolve("workspace");+ const returnedPath = nodePath.join(projectRoot, "src", "returned.ts");+ const callerPath = nodePath.join(projectRoot, "src", "caller.ts");+ const stillDeletedPath = nodePath.join(projectRoot, "src", "still-deleted.ts");+ const deletedFiles = new Map([+ ["src/returned.ts", ["src/caller.ts", "src/missing.ts"]],+ ["src/still-deleted.ts", ["src/other.ts"]],+ ]);++ const plan = planDeletedFileRecovery(+ projectRoot,+ [returnedPath, callerPath],+ deletedFiles,+ );++ expect(plan.recreatedPaths).toEqual([returnedPath]);+ expect(plan.previousDeletedFiles).toEqual(+ new Map([+ [returnedPath, ["src/caller.ts", "src/missing.ts"]],+ [stillDeletedPath, ["src/other.ts"]],+ ]),+ );+ expect(plan.forceReingestPaths).toEqual(new Set([callerPath]));+ expect(plan.nextDeletedFiles).toEqual(+ new Map([[stillDeletedPath, ["src/other.ts"]]]),+ );+ });++ it("deletes removed nodes and their incident edges while preserving current entities", async () => {+ const getPatch = vi+ .fn()+ .mockRejectedValueOnce(new Error("404: not found"))+ .mockResolvedValueOnce({+ data: {+ entityIds: ["kept-node", "removed-node", "kept-edge", "removed-edge"],+ nodeOpCount: 2,+ edgeOpCount: 2,+ },+ });+ const entity = vi.fn().mockResolvedValue({+ node: {},+ claims: [],+ edges: [+ { id: "removed-edge", provenance: { sourceUri: "src/example.ts" } },+ { id: "incoming-edge", provenance: { sourceUri: "src/caller.ts" } },+ { id: "kept-edge", provenance: { sourceUri: "src/example.ts" } },+ ],+ });+ const patch = patchWith([+ { type: "UpsertNode", id: "kept-node", kind: "file", name: "example.ts", attrs: {} },+ {+ type: "UpsertEdge",+ id: "kept-edge",+ src: "kept-node",+ dst: "kept-node",+ predicate: "CONTAINS",+ attrs: {},+ },+ { type: "AssertClaim", entityId: "kept-node", field: "calls:x", value: "x" },+ ]);++ const dependents = new Set<string>();+ const reconciled = await reconcileRemovedEntities(+ { getPatch, entity },+ patch,+ ["missing-patch", "previous-patch"],+ dependents,+ );++ expect(getPatch).toHaveBeenCalledTimes(2);+ expect(entity).toHaveBeenCalledWith("removed-node");+ expect(dependents).toEqual(new Set(["src/caller.ts"]));+ expect(reconciled.ops).toEqual([+ { type: "DeleteNode", id: "removed-node" },+ patch.ops[0],+ { type: "DeleteEdge", id: "removed-edge" },+ { type: "DeleteEdge", id: "incoming-edge" },+ patch.ops[1],+ patch.ops[2],+ ]);+ });++ it("does not repeat edge deletions from a prior tombstone when a file returns", async () => {+ const getPatch = vi.fn().mockResolvedValue({+ data: {+ ops: [+ { type: "DeleteNode", id: "returned-node" },+ { type: "DeleteEdge", id: "incoming-edge" },+ ],+ },+ });+ const entity = vi.fn();+ const patch = patchWith([+ { type: "UpsertNode", id: "returned-node", kind: "file", name: "example.ts" },+ ]);++ const reconciled = await reconcileRemovedEntities(+ { getPatch, entity },+ patch,+ ["tombstone-patch"],+ );++ expect(reconciled.ops).toEqual(patch.ops);+ expect(entity).not.toHaveBeenCalled();+ });++ it("does not delete chunk nodes in map mode", async () => {+ // stripMapModeOps drops chunk UpsertNodes but keeps DeleteNode, and chunk+ // ids hash the start line — so any edit that shifts lines made every+ // downstream chunk look removed. Without this guard, `ix map` after an+ // `ix ingest` deleted the file's chunks and never recreated them.+ const getPatch = vi.fn().mockResolvedValue({+ data: { entityIds: ["file-node", "chunk-L1", "chunk-L10"], nodeOpCount: 3, edgeOpCount: 0 },+ });+ const entity = vi.fn().mockImplementation(async (id: string) => ({+ node: { kind: id.startsWith("chunk") ? "chunk" : "function" },+ claims: [],+ edges: [],+ }));+ const patch = patchWith([+ { type: "UpsertNode", id: "file-node", kind: "file", name: "example.ts" },+ { type: "UpsertNode", id: "chunk-L1", kind: "chunk", name: "a" },+ { type: "UpsertNode", id: "chunk-L12", kind: "chunk", name: "b" },+ ]);++ const reconciled = await reconcileRemovedEntities(+ { getPatch, entity }, patch, ["prev"], undefined, true,+ );++ const deleted = reconciled.ops.filter(o => o.type === "DeleteNode").map(o => o["id"]);+ expect(deleted).toEqual([]);+ });++ it("still deletes chunk nodes outside map mode", async () => {+ const getPatch = vi.fn().mockResolvedValue({+ data: { entityIds: ["file-node", "chunk-L10"], nodeOpCount: 2, edgeOpCount: 0 },+ });+ const entity = vi.fn().mockResolvedValue({ node: { kind: "chunk" }, claims: [], edges: [] });+ const patch = patchWith([+ { type: "UpsertNode", id: "file-node", kind: "file", name: "example.ts" },+ ]);++ const reconciled = await reconcileRemovedEntities({ getPatch, entity }, patch, ["prev"]);++ const deleted = reconciled.ops.filter(o => o.type === "DeleteNode").map(o => o["id"]);+ expect(deleted).toEqual(["chunk-L10"]);+ });++ it("leaves chunk-carrying edges alone in map mode", async () => {+ const getPatch = vi.fn().mockResolvedValue({+ data: { entityIds: ["gone"], nodeOpCount: 1, edgeOpCount: 0 },+ });+ const entity = vi.fn().mockResolvedValue({+ node: { kind: "function" },+ claims: [],+ edges: [+ { id: "contains-chunk", predicate: "CONTAINS_CHUNK", provenance: {} },+ { id: "next-chunk", predicate: "NEXT", provenance: {} },+ { id: "calls-edge", predicate: "CALLS", provenance: {} },+ ],+ });++ const reconciled = await reconcileRemovedEntities(+ { getPatch, entity }, patchWith([]), ["prev"], undefined, true,+ );++ const deletedEdges = reconciled.ops.filter(o => o.type === "DeleteEdge").map(o => o["id"]);+ expect(deletedEdges).toEqual(["calls-edge"]);+ });++ it("fails closed when the previous patch has no entity manifest", async () => {+ const getPatch = vi.fn().mockResolvedValue({ data: {} });+ const entity = vi.fn();++ await expect(+ reconcileRemovedEntities({ getPatch, entity }, patchWith([]), ["previous-patch"]),+ ).rejects.toThrow("has no entity manifest");+ expect(entity).not.toHaveBeenCalled();+ });+});diff --git a/ix-cli/src/cli/__tests__/loadExistingHashes.test.ts b/ix-cli/src/cli/__tests__/loadExistingHashes.test.tsindex f563d969..0f0b5f8f 100644--- a/ix-cli/src/cli/__tests__/loadExistingHashes.test.ts+++ b/ix-cli/src/cli/__tests__/loadExistingHashes.test.ts@@ -52,4 +52,11 @@ describe("loadExistingHashes — workspace-scoped baseline (Ix#225 gap 3)", () =const out = await loadExistingHashes(failing, ["/a.ts"], a => a, () => "ws");expect(out.size).toBe(0);});++ it("propagates lookup failures when deletion cleanup requires a reliable baseline", async () => {+ const failing = { getSourceHashes: async () => { throw new Error("network"); } };+ await expect(+ loadExistingHashes(failing, ["/deleted.ts"], a => a, () => "ws", false, true),+ ).rejects.toThrow("network");+ });});diff --git a/ix-cli/src/cli/__tests__/stale-workspace.test.ts b/ix-cli/src/cli/__tests__/stale-workspace.test.tsindex 53d8b1ca..869704ca 100644--- a/ix-cli/src/cli/__tests__/stale-workspace.test.ts+++ b/ix-cli/src/cli/__tests__/stale-workspace.test.ts@@ -3,10 +3,10 @@ import * as fs from "node:fs";import * as os from "node:os";import * as path from "node:path";-import { ingestMtimeCachePath } from "../config.js";+import { ingestMtimeCachePath, saveConfig } from "../config.js";import { loadIngestBaseline } from "../ingest-baseline.js";import { persistIngestBaselineIfClean } from "../commands/ingest.js";-import { detectStaleFiles } from "../stale.js";+import { detectStaleFiles, isFileStale } from "../stale.js";let home: string;let savedHome: string | undefined;@@ -117,6 +117,89 @@ describe("workspace-scoped staleness", () => {});});+ it("reports a mapped file that was deleted after ingest", async () => {+ const root = path.join(home, "deleted-file");+ const filePath = writeSource(root, "deleted.js", "export const value = 1;\n");+ const ingestedAt = new Date("2026-08-09T18:00:00.000Z");+ persistIngestBaselineIfClean(+ root,+ new Map([[filePath, fs.statSync(filePath).mtimeMs]]),+ 14,+ 0,+ 0,+ ingestedAt,+ );+ saveConfig({+ endpoint: "http://localhost:8090",+ format: "text",+ workspaces: [+ {+ workspace_id: "dead0001",+ workspace_name: "deleted-file",+ root_path: root,+ default: true,+ },+ ],+ });++ fs.unlinkSync(filePath);++ expect(detectStaleFiles(root)).toEqual({+ lastIngestAt: ingestedAt.toISOString(),+ currentRev: 14,+ staleFiles: 1,+ sampleChangedFiles: ["deleted.js"],+ });+ expect(isFileStale(filePath)).toBe(true);+ expect(isFileStale(path.join(root, "never-mapped.js"))).toBe(false);+ });++ it("persists an empty baseline after the last mapped file is deleted", () => {+ const root = path.join(home, "empty-after-delete");+ const filePath = writeSource(root, "deleted.js", "export const value = 1;\n");+ persistIngestBaselineIfClean(+ root,+ new Map([[filePath, fs.statSync(filePath).mtimeMs]]),+ 14,+ 0,+ 0,+ new Date("2026-08-09T18:00:00.000Z"),+ );++ const completedAt = new Date("2026-08-09T18:01:00.000Z");+ const deletedFiles = new Map([[filePath, ["caller.js"]]]);+ expect(+ persistIngestBaselineIfClean(root, new Map(), 15, 0, 0, completedAt, deletedFiles),+ ).toBe(true);++ expect(loadIngestBaseline(root)).toEqual({+ files: new Map(),+ deletedFiles,+ currentRev: 15,+ lastIngestAt: completedAt.toISOString(),+ });+ });++ it("refuses an empty baseline when nothing was deleted", () => {+ // An empty mtime map with no deletions is a discovery failure — an+ // over-broad ignore rule, the wrong cwd — and persisting it silently+ // discards the baseline, forcing a full re-ingest next run.+ const root = path.join(home, "discovery-failure");+ const filePath = writeSource(root, "kept.js", "export const value = 1;\n");+ persistIngestBaselineIfClean(+ root,+ new Map([[filePath, fs.statSync(filePath).mtimeMs]]),+ 20,+ 0,+ 0,+ new Date("2026-08-10T18:00:00.000Z"),+ );++ expect(persistIngestBaselineIfClean(root, new Map(), 21, 0, 0)).toBe(false);+ expect(loadIngestBaseline(root)?.files.size).toBe(1);+ expect(loadIngestBaseline(root)?.currentRev).toBe(20);+ });+it("uses ingest time for files absent from the mtime cache", async () => {const root = path.join(home, "partial-cache");const mappedFile = writeSource(root, "mapped.js", "export const mapped = true;\n");diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 02400ae7..6069ef5b 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -264,6 +264,184 @@ function stripMapModeOps(patch: GraphPatchPayload): GraphPatchPayload {};}+interface StoredPatchEntities {+ nodeIds: string[];+}++function readStoredPatchEntities(raw: unknown): StoredPatchEntities | null {+ const data = (raw as { data?: unknown } | null)?.data as {+ entityIds?: unknown;+ nodeOpCount?: unknown;+ ops?: unknown;+ } | undefined;+ if (!data) return null;+ if (Array.isArray(data.ops)) {+ const ops = data.ops as Array<{ type?: unknown; id?: unknown }>;+ return {+ nodeIds: ops+ .filter(op => op.type === 'UpsertNode' || op.type === 'DeleteNode')+ .map(op => op.id)+ .filter((id): id is string => typeof id === 'string'),+ };+ }+ if (!Array.isArray(data.entityIds)) return null;+ if (!Number.isInteger(data.nodeOpCount)) return null;++ const nodeCount = data.nodeOpCount as number;+ const entityIds = data.entityIds.filter((id): id is string => typeof id === 'string');+ if (nodeCount < 0 || entityIds.length < nodeCount) return null;++ return {+ nodeIds: entityIds.slice(0, nodeCount),+ };+}++async function loadStoredPatchEntities(+ client: Pick<IxClient, 'getPatch'>,+ patchIds: string[],+): Promise<StoredPatchEntities> {+ for (const patchId of patchIds) {+ try {+ const entities = readStoredPatchEntities(await client.getPatch(patchId));+ if (entities) return entities;+ throw new Error(`Patch ${patchId} has no entity manifest`);+ } catch (err) {+ if (!String(err).includes('404:')) throw err;+ }+ }+ throw new Error('Previous source patch was not found');+}++export async function reconcileRemovedEntities(+ client: Pick<IxClient, 'getPatch' | 'entity'>,+ patch: GraphPatchPayload,+ previousPatchIds: string[],+ dependentSourceUris?: Set<string>,+ // `ix map` does not write chunks: stripMapModeOps drops every chunk UpsertNode+ // from the patch. It does NOT drop DeleteNode, and chunk ids hash the chunk's+ // start line, so any edit that shifts lines makes every downstream chunk look+ // removed. Reconciling without this flag therefore had `ix map` after an+ // `ix ingest` delete the file's chunks and never recreate them — the cheaper+ // command silently destroying embedding data the expensive one built. Chunks+ // are not map mode's to reconcile, so in map mode they are left alone.+ mapMode = false,+): Promise<GraphPatchPayload> {+ const previous = await loadStoredPatchEntities(client, previousPatchIds);+ const currentNodeIds = new Set(+ patch.ops+ .filter(op => op.type === 'UpsertNode')+ .map(op => op['id'])+ .filter((id): id is string => typeof id === 'string'),+ );+ const currentEdgeIds = new Set(+ patch.ops+ .filter(op => op.type === 'UpsertEdge')+ .map(op => op['id'])+ .filter((id): id is string => typeof id === 'string'),+ );+ const candidateNodeIds = previous.nodeIds.filter(id => !currentNodeIds.has(id));+ const removedNodeIds: string[] = [];+ const removedEdgeIds = new Set<string>();+ const CHUNK_PREDICATES = new Set(['CONTAINS_CHUNK', 'NEXT']);++ for (const nodeId of candidateNodeIds) {+ try {+ const entity = await client.entity(nodeId);+ // The lookup is already being made for the incident edges, so the kind+ // costs nothing extra. A node whose kind cannot be read (404 — already+ // gone) falls through to the delete, which is a no-op server-side.+ if (mapMode && (entity.node as { kind?: unknown } | undefined)?.kind === 'chunk') {+ continue;+ }+ removedNodeIds.push(nodeId);+ for (const edge of entity.edges ?? []) {+ const edgeRecord = edge as {+ id?: unknown;+ provenance?: { sourceUri?: unknown; source_uri?: unknown };+ };+ const edgeId = edgeRecord.id;+ const predicate = (edge as { predicate?: unknown }).predicate;+ const isChunkEdge = typeof predicate === 'string' && CHUNK_PREDICATES.has(predicate);+ if (typeof edgeId === 'string' && !currentEdgeIds.has(edgeId) && !(mapMode && isChunkEdge)) {+ removedEdgeIds.add(edgeId);+ }+ const sourceUri = edgeRecord.provenance?.sourceUri ?? edgeRecord.provenance?.source_uri;+ if (+ dependentSourceUris &&+ typeof sourceUri === 'string' &&+ sourceUri !== patch.source.uri+ ) {+ dependentSourceUris.add(sourceUri);+ }+ }+ } catch (err) {+ if (!String(err).includes('404:')) throw err;+ // Already absent. Emitting the delete anyway keeps the patch a complete+ // statement of intent and costs nothing.+ removedNodeIds.push(nodeId);+ }+ }++ const nodeOps = patch.ops.filter(op => op.type === 'UpsertNode' || op.type === 'DeleteNode');+ const edgeOps = patch.ops.filter(op => op.type === 'UpsertEdge' || op.type === 'DeleteEdge');+ const otherOps = patch.ops.filter(op =>+ op.type !== 'UpsertNode' && op.type !== 'DeleteNode' &&+ op.type !== 'UpsertEdge' && op.type !== 'DeleteEdge'+ );++ return {+ ...patch,+ ops: [+ ...removedNodeIds.map(id => ({ type: 'DeleteNode', id })),+ ...nodeOps,+ ...[...removedEdgeIds].map(id => ({ type: 'DeleteEdge', id })),+ ...edgeOps,+ ...otherOps,+ ],+ };+}++export function patchRequiresPerFileCommit(patch: GraphPatchPayload): boolean {+ return patch.ops.some(op => op.type === 'DeleteNode' || op.type === 'DeleteEdge');+}++export function planDeletedFileRecovery(+ projectRoot: string,+ filePaths: string[],+ deletedFiles: Map<string, string[]>,+): {+ previousDeletedFiles: Map<string, string[]>;+ nextDeletedFiles: Map<string, string[]>;+ recreatedPaths: string[];+ forceReingestPaths: Set<string>;+} {+ const previousDeletedFiles = new Map<string, string[]>();+ for (const [deletedPath, dependents] of deletedFiles) {+ const absolutePath = nodePath.isAbsolute(deletedPath)+ ? nodePath.resolve(deletedPath)+ : nodePath.resolve(projectRoot, deletedPath);+ previousDeletedFiles.set(absolutePath, dependents);+ }++ const nextDeletedFiles = new Map(previousDeletedFiles);+ const filePathSet = new Set(filePaths.map(filePath => nodePath.resolve(filePath)));+ const recreatedPaths = [...previousDeletedFiles.keys()].filter(filePath =>+ filePathSet.has(filePath)+ );+ const forceReingestPaths = new Set<string>();+ for (const recreatedPath of recreatedPaths) {+ for (const dependent of previousDeletedFiles.get(recreatedPath) ?? []) {+ const absoluteDependent = nodePath.isAbsolute(dependent)+ ? nodePath.resolve(dependent)+ : nodePath.resolve(projectRoot, dependent);+ if (filePathSet.has(absoluteDependent)) forceReingestPaths.add(absoluteDependent);+ }+ nextDeletedFiles.delete(recreatedPath);+ }++ return { previousDeletedFiles, nextDeletedFiles, recreatedPaths, forceReingestPaths };+}+const COMMIT_CONFLICT_RETRY_PATTERNS = ['write-write conflict','timeout waiting to lock key',@@ -310,10 +488,6 @@ async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Pro// Mtime cache — skip readFileSync+sha256 for unchanged files// ----------------------------------------------------------------------------function loadMtimeCache(projectRoot: string): Map<string, number> {- return loadIngestBaseline(projectRoot)?.files ?? new Map();-}-export function persistIngestBaselineIfClean(projectRoot: string,mtimes: Map<string, number>,@@ -321,9 +495,21 @@ export function persistIngestBaselineIfClean(parseErrors: number,commitErrors: number,now?: Date,+ deletedFiles: Map<string, string[]> = new Map(),): boolean {- if (!ingestCompletedCleanly(parseErrors, commitErrors) || mtimes.size === 0) return false;- saveIngestBaseline(projectRoot, mtimes, currentRev, now);+ if (!ingestCompletedCleanly(parseErrors, commitErrors)) return false;+ // An empty mtime map is normally a discovery failure, not an empty repo —+ // an over-broad ignore rule, the wrong cwd, a glob that matched nothing —+ // and persisting it silently discards the baseline, forcing a full re-ingest+ // next run. The original guard rejected every empty map for that reason.+ //+ // But deleting the last mapped file legitimately produces one, and refusing+ // to record that leaves the baseline claiming files that are gone. So the+ // guard now asks *why* the map is empty: with deletions recorded this run,+ // empty is a real state and is persisted; without them, it is still treated+ // as discovery having gone wrong.+ if (mtimes.size === 0 && deletedFiles.size === 0) return false;+ saveIngestBaseline(projectRoot, mtimes, currentRev, now, deletedFiles);return true;}@@ -484,7 +670,17 @@ export async function ingestFiles(const mapMode = opts.mapMode === true;const trueStart = performance.now();- const [{ parseFile, resolveEdges, isGrammarSupported }, { buildPatchWithResolution, fileNodeId, symbolNodeId }, { languageFromPath }] = await loadIngestionModules();+ const [+ { parseFile, resolveEdges, isGrammarSupported },+ {+ buildPatchWithResolution,+ buildDeletionPatch,+ sourcePatchIdCandidates,+ fileNodeId,+ symbolNodeId,+ },+ { languageFromPath },+ ] = await loadIngestionModules();const moduleLoadMs = Math.round(performance.now() - trueStart);@@ -847,7 +1043,20 @@ export async function ingestFiles(const projectRoot = fs.statSync(resolvedPath).isDirectory() ? resolvedPath : nodePath.dirname(resolvedPath);// A just-migrated workspace (new path-based id) has no nodes under the new id, so// skip the mtime pre-filter and re-ingest everything, exactly like --force.- const mtimeCache = (opts.force || workspaceMigrated) ? new Map<string, number>() : loadMtimeCache(projectRoot);+ const previousBaseline = loadIngestBaseline(projectRoot);+ const previousMtimes = previousBaseline?.files ?? new Map<string, number>();+ const {+ previousDeletedFiles,+ nextDeletedFiles,+ forceReingestPaths,+ } = planDeletedFileRecovery(+ projectRoot,+ filePaths,+ previousBaseline?.deletedFiles ?? new Map(),+ );+ const mtimeCache = (opts.force || workspaceMigrated)+ ? new Map<string, number>()+ : new Map(previousMtimes);const currentMtimes = new Map<string, number>();// DB-reset guard: if the mtime cache has entries but the server returns no hashes@@ -871,7 +1080,7 @@ export async function ingestFiles(if (st.size > MAX_FILE_BYTES) { tooLarge++; progressCurrent++; continue; }const mtime = st.mtimeMs;currentMtimes.set(filePath, mtime);- if (!opts.force && mtimeCache.get(filePath) === mtime) {+ if (!opts.force && !forceReingestPaths.has(filePath) && mtimeCache.get(filePath) === mtime) {filesSkipped++;progressCurrent++; // mtime clean — assume unchanged} else {@@ -884,10 +1093,38 @@ export async function ingestFiles(}}- // Phase: hash lookup — only needed when mtime-changed files exist.+ const deletedPaths = workspaceMigrated+ ? []+ : [...previousMtimes.keys()].filter(filePath =>+ !currentMtimes.has(filePath) && !fs.existsSync(filePath)+ );++ // Phase: hash lookup — only needed when files changed or were deleted.let knownHashes: Map<string, string>;- if (opts.force || mtimeChangedPaths.length > 0) {- knownHashes = await loadExistingHashes(client, opts.force ? filePaths : mtimeChangedPaths, toWorkspaceRelative, sourceWorkspaceIdOf, debug);+ const hashLookupPaths = opts.force+ ? [...filePaths, ...deletedPaths]+ : [...mtimeChangedPaths, ...deletedPaths];+ // Deleting from an unreliable baseline is how data gets lost, so the lookup+ // fails closed when deletions are pending — but only the *deletions* need+ // that guarantee. Letting the throw escape aborted the entire map over one+ // transient blip, taking the ordinary ingest of every changed file with it.+ // So catch it here: drop the deletions for this run (they stay in the+ // baseline and are retried next time) and let the rest proceed.+ if (hashLookupPaths.length > 0) {+ try {+ knownHashes = await loadExistingHashes(+ client,+ hashLookupPaths,+ toWorkspaceRelative,+ sourceWorkspaceIdOf,+ debug,+ deletedPaths.length > 0,+ );+ } catch (err) {+ if (debug) process.stderr.write(`\n [deletion cleanup skipped] hash lookup failed: ${err}\n`);+ deletedPaths.length = 0;+ knownHashes = new Map();+ }if (debug) process.stderr.write(`\n Source hash lookup: ${knownHashes.size} known hashes (${mtimeChangedPaths.length} mtime-changed)\n`);} else {// All files are mtime-clean — skip server round-trip entirely.@@ -952,14 +1189,30 @@ export async function ingestFiles(const commitPreparedPatches = async (preparedPatches: PreparedPatch[],totalFiles: number,- opts?: { updateProgress?: boolean },+ opts?: {+ updateProgress?: boolean;+ onCommitted?: (item: PreparedPatch, rev: number) => void;+ },): Promise<number> => {if (preparedPatches.length === 0) return 0;const chunks: PreparedPatch[][] = [];- for (let i = 0; i < preparedPatches.length; i += COMMIT_HTTP_MAX_FILES) {- chunks.push(preparedPatches.slice(i, i + COMMIT_HTTP_MAX_FILES));+ let bulkChunk: PreparedPatch[] = [];+ const flushBulkChunk = (): void => {+ if (bulkChunk.length === 0) return;+ chunks.push(bulkChunk);+ bulkChunk = [];+ };+ for (const item of preparedPatches) {+ if (patchRequiresPerFileCommit(item.patch)) {+ flushBulkChunk();+ chunks.push([item]);+ } else {+ bulkChunk.push(item);+ if (bulkChunk.length === COMMIT_HTTP_MAX_FILES) flushBulkChunk();+ }}+ flushBulkChunk();const commitMsPerChunk = new Array<number>(chunks.length).fill(0);let nextChunk = 0;@@ -973,24 +1226,9 @@ export async function ingestFiles(const endFile = chunk[chunk.length - 1].fileNumber;const endingPath = nodePath.basename(chunk[chunk.length - 1].filePath);const patches = chunk.map(item => item.patch);-- try {- setCurrentWork(`commit ${startFile}-${endFile} of ${totalFiles} ending ${endingPath}`);- const commitStart = performance.now();- const result = await retryOnConflict(() => client.commitPatchBulk(patches), COMMIT_CONFLICT_RETRIES);- const chunkMs = Math.round(performance.now() - commitStart);- commitMsPerChunk[ci] = chunkMs;- timings.bulkCommitMs += chunkMs;- if (result.rev > latestRev) latestRev = result.rev;- patchesApplied += patches.length;- } catch (err) {- if (debug) {- process.stderr.write(- `\n [bulk failed, falling back to per-file] files ${startFile}-${endFile} (${patches.length} patches): ${err}\n`- );- }+ const commitIndividually = async (): Promise<void> => {// Chain this chunk's fallback work onto the shared tail so that only one- // fallback loop runs at a time. This prevents concurrent commitPatch+ // per-file loop runs at a time. This prevents concurrent commitPatch// transactions from racing on revisions.current (write-write conflict).const prev = fallbackTail;let resolveThis!: () => void;@@ -1006,6 +1244,7 @@ export async function ingestFiles(timings.fallbackCommitMs += chunkMs;if (result.rev > latestRev) latestRev = result.rev;patchesApplied++;+ opts?.onCommitted?.(item, result.rev);} catch (commitErr) {// Counted apart from parseErrors: a patch that parsed fine and// failed to commit means the graph is now behind the working@@ -1023,6 +1262,31 @@ export async function ingestFiles(} finally {resolveThis();}+ };++ const hasDeletion = patches.some(patchRequiresPerFileCommit);++ if (hasDeletion) {+ await commitIndividually();+ } else {+ try {+ setCurrentWork(`commit ${startFile}-${endFile} of ${totalFiles} ending ${endingPath}`);+ const commitStart = performance.now();+ const result = await retryOnConflict(() => client.commitPatchBulk(patches), COMMIT_CONFLICT_RETRIES);+ const chunkMs = Math.round(performance.now() - commitStart);+ commitMsPerChunk[ci] = chunkMs;+ timings.bulkCommitMs += chunkMs;+ if (result.rev > latestRev) latestRev = result.rev;+ patchesApplied += patches.length;+ for (const item of chunk) opts?.onCommitted?.(item, result.rev);+ } catch (err) {+ if (debug) {+ process.stderr.write(+ `\n [bulk failed, falling back to per-file] files ${startFile}-${endFile} (${patches.length} patches): ${err}\n`+ );+ }+ await commitIndividually();+ }}if (opts?.updateProgress) {@@ -1078,7 +1342,17 @@ export async function ingestFiles(for (let j = 0; j < batch.length; j++) {const { parsed: p, hash, previousHash } = batch[j];try {- let patch = buildPatchFn!(p, hash, fileWorkspaceId(p.filePath), batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));+ const fileWorkspace = fileWorkspaceId(p.filePath);+ let patch = buildPatchFn!(p, hash, fileWorkspace, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));+ if (previousHash) {+ patch = await reconcileRemovedEntities(+ client,+ patch,+ sourcePatchIdCandidates(p.filePath, previousHash, fileWorkspace),+ undefined,+ mapMode,+ );+ }if (mapMode) patch = stripMapModeOps(patch);// source.uri (workspace-relative) and source.workspaceId are set// inside buildPatch; the backend stores both as opaque attributes.@@ -1147,7 +1421,17 @@ export async function ingestFiles(for (let j = 0; j < allParsed.length; j++) {const { parsed: p, hash, previousHash } = allParsed[j];try {- let patch = buildPatchFn!(p, hash, fileWorkspaceId(p.filePath), edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));+ const fileWorkspace = fileWorkspaceId(p.filePath);+ let patch = buildPatchFn!(p, hash, fileWorkspace, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));+ if (previousHash) {+ patch = await reconcileRemovedEntities(+ client,+ patch,+ sourcePatchIdCandidates(p.filePath, previousHash, fileWorkspace),+ undefined,+ mapMode,+ );+ }if (mapMode) patch = stripMapModeOps(patch);// source.uri and source.workspaceId are set inside buildPatch (see flushBatch).preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath));@@ -1192,7 +1476,11 @@ export async function ingestFiles(try {const bytes = fs.readFileSync(filePath);const hash = sha256(bytes);- if (knownHashes.get(filePath) === hash) { filesSkipped++; progressCurrent++; continue; }+ if (!forceReingestPaths.has(filePath) && knownHashes.get(filePath) === hash) {+ filesSkipped++;+ progressCurrent++;+ continue;+ }const previousHash = knownHashes.get(filePath);changedPaths.push({ filePath, bytes, hash, previousHash: previousHash !== hash ? previousHash : undefined });} catch (err) {@@ -1356,7 +1644,10 @@ export async function ingestFiles(await fh.close();}const hash = sha256(bytes);- if (!opts.force && knownHashes.get(absFilePath) === hash) { filesSkipped++; return; }+ if (!opts.force && !forceReingestPaths.has(absFilePath) && knownHashes.get(absFilePath) === hash) {+ filesSkipped++;+ return;+ }const previousHash = knownHashes.get(absFilePath);const sourceText = bytes.toString('utf-8');if (isLikelyMinifiedSource(sourceText)) {@@ -1397,6 +1688,80 @@ export async function ingestFiles(// Clear the parse barif (interval) process.stderr.write('\r' + ' '.repeat(PROG_LINE_WIDTH) + '\r');+ if (deletedPaths.length > 0) {+ progressPhase = 'Saving';+ progressTotal = deletedPaths.length;+ progressCurrent = 0;+ const deletedPatches: PreparedPatch[] = [];+ const durableDeletedFiles = new Map(previousDeletedFiles);+ const pendingDeletionRecovery = new Map<+ string,+ { absolutePath: string; dependents: string[] }+ >();++ for (let i = 0; i < deletedPaths.length; i++) {+ const absFilePath = deletedPaths[i];+ const previousHash = knownHashes.get(absFilePath);+ if (!previousHash) {+ nextDeletedFiles.set(absFilePath, []);+ progressCurrent++;+ continue;+ }++ const relFilePath = toWorkspaceRelative(absFilePath);+ const fileWorkspace = sourceWorkspaceIdOf(absFilePath);+ const dependentSourceUris = new Set(nextDeletedFiles.get(absFilePath) ?? []);+ try {+ let patch = buildDeletionPatch(+ relFilePath,+ previousHash,+ previousBaseline?.lastIngestAt ?? String(previousMtimes.get(absFilePath) ?? ''),+ fileWorkspace,+ [],+ fileMultiRepo(relFilePath),+ );+ patch = await reconcileRemovedEntities(+ client,+ patch,+ sourcePatchIdCandidates(relFilePath, previousHash, fileWorkspace),+ dependentSourceUris,+ mapMode,+ );+ if (mapMode) patch = stripMapModeOps(patch);+ deletedPatches.push(makePreparedPatch(patch, i + 1, relFilePath));+ const dependents = [...dependentSourceUris].sort();+ nextDeletedFiles.set(absFilePath, dependents);+ pendingDeletionRecovery.set(relFilePath, {+ absolutePath: absFilePath,+ dependents,+ });+ } catch (err) {+ commitErrors++;+ if (debug) process.stderr.write(`\n [deletion error] ${relFilePath}: ${err}\n`);+ }+ progressCurrent++;+ }++ if (deletedPatches.length > 0) {+ filesChanged += deletedPatches.length;+ await commitPreparedPatches(deletedPatches, deletedPaths.length, {+ updateProgress: true,+ onCommitted: item => {+ const recovery = pendingDeletionRecovery.get(item.filePath);+ if (!recovery || !previousBaseline) return;+ durableDeletedFiles.set(recovery.absolutePath, recovery.dependents);+ saveIngestBaseline(+ projectRoot,+ previousMtimes,+ previousBaseline.currentRev,+ new Date(previousBaseline.lastIngestAt),+ durableDeletedFiles,+ );+ },+ });+ }+ }+const committed = performance.now();// Persist mtime cache so next run can skip unchanged files quickly.@@ -1412,6 +1777,8 @@ export async function ingestFiles(latestRev,parseErrors,commitErrors,+ undefined,+ nextDeletedFiles,);// Migration cleanup (Ix#225 gap 2): the re-ingest under the new path-based id has@@ -1656,6 +2023,7 @@ export async function loadExistingHashes(toRelative: (absPath: string) => string,workspaceIdOf: (absPath: string) => string,debug = false,+ throwOnError = false,): Promise<Map<string, string>> {try {// Each file is matched against its OWN workspace's stored hash. The server keys@@ -1677,6 +2045,7 @@ export async function loadExistingHashes(return out;} catch (err) {if (debug) process.stderr.write(`\n [hash lookup failed] ${err}\n`);+ if (throwOnError) throw err;return new Map();}}diff --git a/ix-cli/src/cli/commands/ingestion-loader.ts b/ix-cli/src/cli/commands/ingestion-loader.tsindex b448f8f2..b89d1beb 100644--- a/ix-cli/src/cli/commands/ingestion-loader.ts+++ b/ix-cli/src/cli/commands/ingestion-loader.ts@@ -11,6 +11,8 @@ type IngestionModule = {type PatchBuilderModule = {buildPatch: (parsed: any, hash: string, workspaceId: string, previousSourceHash?: string) => any;buildPatchWithResolution: (parsed: any, hash: string, workspaceId: string, resolvedEdges: any[], previousSourceHash?: string) => any;+ buildDeletionPatch: (filePath: string, previousSourceHash: string, deletionToken: string, workspaceId: string, ops: any[], multiRepo?: any) => any;+ sourcePatchIdCandidates: (filePath: string, sourceHash: string, workspaceId: string) => string[];fileNodeId: (workspaceId: string, filePath: string) => string;symbolNodeId: (workspaceId: string, filePath: string, qualifiedKey: string) => string;};diff --git a/ix-cli/src/cli/ingest-baseline.ts b/ix-cli/src/cli/ingest-baseline.tsindex 8f42b401..ff807511 100644--- a/ix-cli/src/cli/ingest-baseline.ts+++ b/ix-cli/src/cli/ingest-baseline.ts@@ -5,12 +5,14 @@ import { ingestMtimeCachePath } from "./config.js";interface SerializedIngestBaseline {root: string;files: Record<string, number>;+ deletedFiles?: Record<string, string[]>;currentRev?: number;lastIngestAt?: string;}export interface IngestBaseline {files: Map<string, number>;+ deletedFiles: Map<string, string[]>;currentRev: number;lastIngestAt: string;}@@ -25,6 +27,14 @@ export function loadIngestBaseline(projectRoot: string): IngestBaseline | null {for (const [filePath, mtime] of Object.entries(data.files)) {if (Number.isFinite(mtime)) files.set(filePath, mtime);}+ const deletedFiles = new Map<string, string[]>();+ for (const [filePath, dependents] of Object.entries(data.deletedFiles ?? {})) {+ if (!Array.isArray(dependents)) continue;+ deletedFiles.set(+ filePath,+ dependents.filter((dependent): dependent is string => typeof dependent === "string"),+ );+ }const parsedTimestamp = data.lastIngestAt ? Date.parse(data.lastIngestAt) : Number.NaN;const lastIngestAt = Number.isFinite(parsedTimestamp)@@ -37,7 +47,7 @@ export function loadIngestBaseline(projectRoot: string): IngestBaseline | null {? data.currentRev: 0;- return { files, currentRev, lastIngestAt };+ return { files, deletedFiles, currentRev, lastIngestAt };} catch {return null;}@@ -48,12 +58,14 @@ export function saveIngestBaseline(mtimes: Map<string, number>,currentRev: number,now: Date = new Date(),+ deletedFiles: Map<string, string[]> = new Map(),): void {try {const previousRev = loadIngestBaseline(projectRoot)?.currentRev ?? 0;const data: SerializedIngestBaseline = {root: projectRoot,files: Object.fromEntries(mtimes),+ deletedFiles: Object.fromEntries(deletedFiles),currentRev: currentRev > 0 ? currentRev : previousRev,lastIngestAt: now.toISOString(),};diff --git a/ix-cli/src/cli/stale.ts b/ix-cli/src/cli/stale.tsindex 7c3bef04..a0512135 100644--- a/ix-cli/src/cli/stale.ts+++ b/ix-cli/src/cli/stale.ts@@ -93,6 +93,7 @@ export function detectStaleFiles(}const files = collectFiles(workspaceRoot);+ const currentFiles = new Set(files.map((filePath) => path.resolve(filePath)));const changedFiles: string[] = [];for (const filePath of files) {@@ -110,6 +111,15 @@ export function detectStaleFiles(}}+ for (const ingestedPath of baseline.files.keys()) {+ const absolutePath = path.isAbsolute(ingestedPath)+ ? path.resolve(ingestedPath)+ : path.resolve(workspaceRoot, ingestedPath);+ if (!currentFiles.has(absolutePath) && !fs.existsSync(absolutePath)) {+ changedFiles.push(path.relative(workspaceRoot, absolutePath));+ }+ }+return {lastIngestAt: baseline.lastIngestAt,currentRev: baseline.currentRev,@@ -126,14 +136,18 @@ export function detectStaleFiles(* this used to need was once a backend round-trip per file.*/export function isFileStale(filePath: string): boolean {- if (!fs.existsSync(filePath)) return false;-const workspaceRoot = path.resolve(resolveWorkspaceRoot());const baseline = loadIngestBaseline(workspaceRoot);if (!baseline) return false;+ const absolutePath = path.isAbsolute(filePath)+ ? path.resolve(filePath)+ : path.resolve(workspaceRoot, filePath);+ if (!fs.existsSync(absolutePath)) {+ return baseline.files.has(absolutePath) || baseline.files.has(filePath);+ }+try {- const absolutePath = path.resolve(filePath);return differsFromIngestBaseline(absolutePath,fs.statSync(absolutePath).mtimeMs,