Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): delete-bearing patches go through bulk; reconcile concurrently
KageBinary4d1273d098e2
diff --git a/ix-cli/src/cli/__tests__/helpers/fake-backend.ts b/ix-cli/src/cli/__tests__/helpers/fake-backend.tsindex 1c007929..75421a4d 100644--- a/ix-cli/src/cli/__tests__/helpers/fake-backend.ts+++ b/ix-cli/src/cli/__tests__/helpers/fake-backend.ts@@ -116,6 +116,12 @@ export class FakeBackend {* uri). An older id is written again and becomes the head (BEW-03).*/semantics: "legacy" | "head" | undefined;+ /**+ * The `release_version` /v1/health reports. Unset, `head` reports a release+ * whose bulk route applies deletes (BEW-03 and #237 are in it) and the others+ * report none, as a pre-#157 backend does. `null` reports none in any mode.+ */+ releaseVersion: string | null | undefined = undefined;private readonly nodes = new Map<string, { kind: unknown; name: unknown; live: boolean }>();private readonly edges = new Map<string, {src: unknown; dst: unknown; predicate: unknown; workspaceId: string; uri: string; live: boolean;@@ -158,21 +164,18 @@ export class FakeBackend {return this.heads.get(`${record.workspaceId}\0${record.uri}`) === patchId;}- private applyPatch(patch: SentPatch, sweep: boolean, bulkGroup: string | null): void {+ private sweepEdgesOf(patch: SentPatch): void {const workspaceId = patch.source?.workspaceId ?? "";const uri = patch.source?.uri ?? "";- this.rev++;- if (sweep) {- for (const e of this.edges.values()) {- if (e.live && e.uri === uri && e.workspaceId === workspaceId) e.live = false;- }+ for (const e of this.edges.values()) {+ if (e.live && e.uri === uri && e.workspaceId === workspaceId) e.live = false;}+ }++ private applyDeletes(patch: SentPatch): void {for (const op of patch.ops ?? []) {const id = String(op.id);- if (op.type === "UpsertNode") this.nodes.set(id, { kind: op.kind, name: op.name, live: true });- else if (op.type === "UpsertEdge") {- this.edges.set(id, { src: op.src, dst: op.dst, predicate: op.predicate, workspaceId, uri, live: true });- } else if (op.type === "DeleteNode") {+ if (op.type === "DeleteNode") {const n = this.nodes.get(id);if (n) n.live = false;} else if (op.type === "DeleteEdge") {@@ -180,6 +183,26 @@ export class FakeBackend {if (e) e.live = false;}}+ }++ /**+ * One patch's writes. A bulk request runs its phases across all of its+ * patches -- every sweep, then every delete, then every upsert -- as+ * BulkWriteApi `commitChunk` does, so a delete in one file's patch never+ * removes what another patch of the same request writes.+ */+ private applyPatch(patch: SentPatch, bulkGroup: string | null, deletesApplied = false): void {+ const workspaceId = patch.source?.workspaceId ?? "";+ const uri = patch.source?.uri ?? "";+ this.rev++;+ if (!deletesApplied) this.applyDeletes(patch);+ for (const op of patch.ops ?? []) {+ const id = String(op.id);+ if (op.type === "UpsertNode") this.nodes.set(id, { kind: op.kind, name: op.name, live: true });+ else if (op.type === "UpsertEdge") {+ this.edges.set(id, { src: op.src, dst: op.dst, predicate: op.predicate, workspaceId, uri, live: true });+ }+ }if (patch.patchId === undefined) return;this.stored.set(patch.patchId, {rev: this.rev, workspaceId, uri, hash: patch.source?.sourceHash, ops: patch.ops ?? [], bulkGroup,@@ -193,14 +216,16 @@ export class FakeBackend {if (patch?.patchId !== undefined && this.alreadyCommitted(patch.patchId)) {return send(200, { rev: this.stored.get(patch.patchId)!.rev, status: "Idempotent" });}- if (patch) this.applyPatch(patch, false, null);+ if (patch) this.applyPatch(patch, null);return send(200, { rev: this.rev, status: "Ok" });}const ids = patches.map((p) => p.patchId ?? "");const group = createHash("sha256").update(ids.join(",")).digest("hex");const done = ids.filter((id) => this.alreadyCommitted(id));if (done.length === 0) {- for (const p of patches) this.applyPatch(p, true, group);+ for (const p of patches) this.sweepEdgesOf(p);+ for (const p of patches) this.applyDeletes(p);+ for (const p of patches) this.applyPatch(p, group, true);return send(200, { rev: this.rev, applied: patches.length, status: "Ok" });}if (done.length !== ids.length) {@@ -419,7 +444,10 @@ export class FakeBackend {return;}- if (path === "/v1/health") return send(200, { status: "ok", version: "1.0.28" });+ if (path === "/v1/health") {+ const release = this.releaseVersion !== undefined ? this.releaseVersion : this.semantics === "head" ? "1.0.34" : null;+ return send(200, { status: "ok", version: "1.0.28", ...(release ? { release_version: release } : {}) });+ }if (path === "/v1/source-hashes") {if (this.failSourceHashes) return send(500, { error: "500: transaction begin timeout" });let uris: string[] = [];diff --git a/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts b/ix-cli/src/cli/__tests__/ingest-equivalence.test.tsindex a3ed11d5..df47bb61 100644--- a/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts@@ -264,10 +264,15 @@ describe("incremental ix map equals a fresh map (fake backend)", () => {});// F-04. Dropping the file's last import removes a node, the patch carries a- // DeleteNode, and a delete-bearing patch goes to `/v1/patch`, which does not- // sweep: the removed CALLS and IMPORTS edges stay live.- it.fails("IN-04: removing a file's last call retires its edges", async () => {- await runSequence("head", [edit("removeCall", "web/main.ts")]);+ // DeleteNode, and a delete-bearing patch went to `/v1/patch`, which does not+ // sweep: the removed CALLS and IMPORTS edges stayed live. A backend whose+ // bulk route applies deletes now takes it there, and nothing goes per file.+ it("IN-04: removing a file's last call retires its edges, with no /v1/patch", async () => {+ await runSequence("head", [edit("removeCall", "web/main.ts")], {+ check: (world) => {+ expect(world.backend.requests.filter((r) => r.path === "/v1/patch"), "per-file commits").toEqual([]);+ },+ });});// F-03. Files that call a renamed or deleted function are unchanged, sodiff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.tsindex 831d103d..3ebb83d7 100644--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts@@ -1161,6 +1161,40 @@ describe("ingestFiles against a fake backend", () => {expect(backend.singleCount, "the fan-out is for failures only").toBe(0);});+ describe("a patch that deletes (IN-04)", () => {+ const incremental = () =>+ ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });+ const perFile = () => backend.requests.filter(r => r.path === "/v1/patch").length;+ /** Map two files, then drop `g` from one of them: its patch carries a DeleteNode. */+ const removeAFunction = async (): Promise<void> => {+ backend.semantics = "head";+ mkdirSync(join(repo, "src"), { recursive: true });+ writeFileSync(join(repo, "src", "a.ts"), "export function f(): number { return 1; }\nexport function g(): number { return 2; }\n", "utf8");+ writeFileSync(join(repo, "src", "b.ts"), "export const b = 1;\n", "utf8");+ execFileSync("git", ["init", "-q"], { cwd: repo, stdio: "ignore" });+ execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });+ await incremental();+ writeFileSync(join(repo, "src", "a.ts"), "export function f(): number { return 1; }\n", "utf8");+ backend.resetRequests();+ await incremental();+ };++ it("goes through the bulk route on a backend that applies bulk deletes", async () => {+ backend.releaseVersion = "1.0.32";+ await removeAFunction();+ expect(backend.lastOps.get("src/a.ts")?.some(op => op.type === "DeleteNode"), "the patch deletes").toBe(true);+ expect(backend.bulkCount).toBe(1);+ expect(perFile()).toBe(0);+ });++ it("goes to /v1/patch on its own on a backend that reports no release, or an older one", async () => {+ backend.releaseVersion = null;+ await removeAFunction();+ expect(backend.lastOps.get("src/a.ts")?.some(op => op.type === "DeleteNode"), "the patch deletes").toBe(true);+ expect(perFile()).toBe(1);+ });+ });+// ── Which workspace a path belongs to ─────────────────────────────────//// `ix ingest <path>` used to treat the path as its own workspace root: `ixdiff --git a/ix-cli/src/cli/__tests__/ingest-reconcile.test.ts b/ix-cli/src/cli/__tests__/ingest-reconcile.test.tsindex 54b3b280..4211fca8 100644--- a/ix-cli/src/cli/__tests__/ingest-reconcile.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-reconcile.test.ts@@ -5,10 +5,13 @@ import * as nodePath from "node:path";import type { GraphPatchPayload } from "../../client/types.js";import {+ BULK_DELETES_SINCE,+ backendAppliesBulkDeletes,patchRequiresPerFileCommit,planDeletedFileRecovery,reconcileRemovedEntities,} from "../commands/ingest.js";+import { createLimiter } from "../tree-walk.js";function patchWith(ops: GraphPatchPayload["ops"]): GraphPatchPayload {return {@@ -29,10 +32,72 @@ function patchWith(ops: GraphPatchPayload["ops"]): GraphPatchPayload {}describe("reconcileRemovedEntities", () => {- it("routes patches with deletion ops away from the bulk endpoint", () => {+ it("routes patches with deletion ops away from the bulk endpoint, unless it applies them", () => {expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteNode", id: "old-node" }]))).toBe(true);expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteEdge", id: "old-edge" }]))).toBe(true);expect(patchRequiresPerFileCommit(patchWith([{ type: "UpsertNode", id: "node" }]))).toBe(false);+ expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteNode", id: "old-node" }]), true)).toBe(false);+ expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteEdge", id: "old-edge" }]), true)).toBe(false);+ });++ it("trusts the bulk route with deletes from the release that applies them", () => {+ const env = {};+ expect(backendAppliesBulkDeletes({ release_version: BULK_DELETES_SINCE }, env)).toBe(true);+ expect(backendAppliesBulkDeletes({ release_version: "1.0.34" }, env)).toBe(true);+ expect(backendAppliesBulkDeletes({ release_version: "v1.1.0" }, env)).toBe(true);+ expect(backendAppliesBulkDeletes({ release_version: "1.0.31" }, env)).toBe(false);+ expect(backendAppliesBulkDeletes({ release_version: "0.9.99" }, env)).toBe(false);+ // No release, or not a version: an image the release pipeline did not build.+ expect(backendAppliesBulkDeletes({}, env)).toBe(false);+ expect(backendAppliesBulkDeletes({ release_version: "dev" }, env)).toBe(false);+ expect(backendAppliesBulkDeletes(null, env)).toBe(false);+ expect(backendAppliesBulkDeletes({}, { IX_BULK_DELETES: "1" })).toBe(true);+ expect(backendAppliesBulkDeletes({ release_version: "1.0.34" }, { IX_BULK_DELETES: "0" })).toBe(false);+ });++ it("looks up no chunk the stored patch names as one, in map mode", async () => {+ const getPatch = vi.fn().mockResolvedValue({+ data: {+ ops: [+ { type: "UpsertNode", id: "file-node", kind: "file" },+ { type: "UpsertNode", id: "chunk-L10", kind: "chunk" },+ { type: "UpsertNode", id: "gone-fn", kind: "function" },+ ],+ },+ });+ const entity = vi.fn().mockResolvedValue({ node: { kind: "function" }, claims: [], edges: [] });+ const patch = patchWith([{ type: "UpsertNode", id: "file-node", kind: "file", name: "example.ts" }]);++ const reconciled = await reconcileRemovedEntities({ getPatch, entity }, patch, ["prev"], undefined, true);++ expect(entity.mock.calls.map(c => c[0])).toEqual(["gone-fn"]);+ expect(reconciled.ops.filter(o => o.type === "DeleteNode").map(o => o["id"])).toEqual(["gone-fn"]);+ });++ it("runs the entity lookups within the limit, and keeps candidate order whatever finishes first", async () => {+ const ids = Array.from({ length: 20 }, (_, i) => `n${i}`);+ const getPatch = vi.fn().mockResolvedValue({+ data: { ops: ids.map(id => ({ type: "UpsertNode", id, kind: "function" })) },+ });+ let active = 0;+ let peak = 0;+ const entity = vi.fn().mockImplementation(async (id: string) => {+ active++;+ peak = Math.max(peak, active);+ // Later ids finish first.+ await new Promise(r => setTimeout(r, 40 - Number(id.slice(1)) * 2));+ active--;+ return { node: {}, claims: [], edges: [{ id: `e-${id}`, provenance: { sourceUri: "src/caller.ts" } }] };+ });+ const patch = patchWith([]);++ const reconciled = await reconcileRemovedEntities(+ { getPatch, entity }, patch, ["prev"], undefined, false, createLimiter(4),+ );++ expect(peak).toBe(4);+ expect(reconciled.ops.filter(o => o.type === "DeleteNode").map(o => o["id"])).toEqual(ids);+ expect(reconciled.ops.filter(o => o.type === "DeleteEdge").map(o => o["id"])).toEqual(ids.map(id => `e-${id}`));});it("reingests surviving dependents when a deleted file returns", () => {diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex b7c7e070..bc04dc3e 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -55,6 +55,8 @@ import {import { printJson } from '../format.js';import { describeReplayedChanges } from '../graph-health.js';import { createClient } from "../../client/factory.js";+import { mapLimit } from "../../client/request-memo.js";+import { createLimiter, type Limiter } from "../tree-walk.js";// ---------------------------------------------------------------------------// File discovery// ---------------------------------------------------------------------------@@ -301,6 +303,8 @@ export function stripMapModeOps(patch: GraphPatchPayload): GraphPatchPayload {interface StoredPatchEntities {nodeIds: string[];+ /** Node kinds the stored patch wrote, when it kept its ops; empty otherwise. */+ kinds: Map<string, string>;}function readStoredPatchEntities(raw: unknown): StoredPatchEntities | null {@@ -311,12 +315,17 @@ function readStoredPatchEntities(raw: unknown): StoredPatchEntities | null {} | undefined;if (!data) return null;if (Array.isArray(data.ops)) {- const ops = data.ops as Array<{ type?: unknown; id?: unknown }>;+ const ops = data.ops as Array<{ type?: unknown; id?: unknown; kind?: unknown }>;+ const kinds = new Map<string, string>();+ for (const op of ops) {+ if (op.type === 'UpsertNode' && typeof op.id === 'string' && typeof op.kind === 'string') kinds.set(op.id, op.kind);+ }return {nodeIds: ops.filter(op => op.type === 'UpsertNode' || op.type === 'DeleteNode').map(op => op.id).filter((id): id is string => typeof id === 'string'),+ kinds,};}if (!Array.isArray(data.entityIds)) return null;@@ -328,6 +337,7 @@ function readStoredPatchEntities(raw: unknown): StoredPatchEntities | null {return {nodeIds: entityIds.slice(0, nodeCount),+ kinds: new Map(),};}@@ -360,6 +370,9 @@ export async function reconcileRemovedEntities(// command silently destroying embedding data the expensive one built. Chunks// are not map mode's to reconcile, so in map mode they are left alone.mapMode = false,+ // Bounds the entity lookups. One limiter shared by every file a run+ // reconciles keeps the whole run to its slots, however many files run at once.+ lookupLimit: Limiter = task => task(),): Promise<GraphPatchPayload> {const previous = await loadStoredPatchEntities(client, previousPatchIds);const currentNodeIds = new Set(@@ -374,46 +387,57 @@ export async function reconcileRemovedEntities(.map(op => op['id']).filter((id): id is string => typeof id === 'string'),);- const candidateNodeIds = previous.nodeIds.filter(id => !currentNodeIds.has(id));+ // A chunk the stored patch names as one is skipped in map mode without a+ // lookup; patches stored without their ops still need the lookup for it.+ const candidateNodeIds = previous.nodeIds.filter(id =>+ !currentNodeIds.has(id) && !(mapMode && previous.kinds.get(id) === 'chunk'));const removedNodeIds: string[] = [];const removedEdgeIds = new Set<string>();const CHUNK_PREDICATES = new Set(['CONTAINS_CHUNK', 'NEXT']);- for (const nodeId of candidateNodeIds) {+ // The lookups run concurrently; their results are walked in candidate order,+ // so the patch comes out the same however they complete.+ const lookups = await Promise.all(candidateNodeIds.map(nodeId => lookupLimit(async () => {try {- const entity = await client.entity(nodeId);- // The lookup is already being made for the incident edges, so the kind- // costs nothing extra. A node whose kind cannot be read (404 — already- // gone) falls through to the delete, which is a no-op server-side.- if (mapMode && (entity.node as { kind?: unknown } | undefined)?.kind === 'chunk') {- continue;- }- removedNodeIds.push(nodeId);- for (const edge of entity.edges ?? []) {- const edgeRecord = edge as {- id?: unknown;- provenance?: { sourceUri?: unknown; source_uri?: unknown };- };- const edgeId = edgeRecord.id;- const predicate = (edge as { predicate?: unknown }).predicate;- const isChunkEdge = typeof predicate === 'string' && CHUNK_PREDICATES.has(predicate);- if (typeof edgeId === 'string' && !currentEdgeIds.has(edgeId) && !(mapMode && isChunkEdge)) {- removedEdgeIds.add(edgeId);- }- const sourceUri = edgeRecord.provenance?.sourceUri ?? edgeRecord.provenance?.source_uri;- if (- dependentSourceUris &&- typeof sourceUri === 'string' &&- sourceUri !== patch.source.uri- ) {- dependentSourceUris.add(sourceUri);- }- }+ return { nodeId, entity: await client.entity(nodeId) };} catch (err) {if (!isNotFoundError(err)) throw err;+ return { nodeId, entity: null };+ }+ })));++ for (const { nodeId, entity } of lookups) {+ if (entity === null) {// Already absent. Emitting the delete anyway keeps the patch a complete// statement of intent and costs nothing.removedNodeIds.push(nodeId);+ continue;+ }+ // The lookup is already being made for the incident edges, so the kind+ // costs nothing extra.+ if (mapMode && (entity.node as { kind?: unknown } | undefined)?.kind === 'chunk') {+ continue;+ }+ removedNodeIds.push(nodeId);+ for (const edge of entity.edges ?? []) {+ const edgeRecord = edge as {+ id?: unknown;+ provenance?: { sourceUri?: unknown; source_uri?: unknown };+ };+ const edgeId = edgeRecord.id;+ const predicate = (edge as { predicate?: unknown }).predicate;+ const isChunkEdge = typeof predicate === 'string' && CHUNK_PREDICATES.has(predicate);+ if (typeof edgeId === 'string' && !currentEdgeIds.has(edgeId) && !(mapMode && isChunkEdge)) {+ removedEdgeIds.add(edgeId);+ }+ const sourceUri = edgeRecord.provenance?.sourceUri ?? edgeRecord.provenance?.source_uri;+ if (+ dependentSourceUris &&+ typeof sourceUri === 'string' &&+ sourceUri !== patch.source.uri+ ) {+ dependentSourceUris.add(sourceUri);+ }}}@@ -436,10 +460,56 @@ export async function reconcileRemovedEntities(};}-export function patchRequiresPerFileCommit(patch: GraphPatchPayload): boolean {+/**+ * Whether `patch` must go to `/v1/patch` on its own. Only a patch that deletes,+ * and only against a backend whose bulk route is not known to apply deletes.+ * `/v1/patch` does not sweep the file's old edges, so a removed call between+ * two surviving symbols stayed live there; the bulk route sweeps them.+ */+/**+ * Files reconciled at once, and entity lookups in flight across all of them:+ * the graph walkers' cap (`EXPAND_CONCURRENCY`), for the same reason.+ */+const RECONCILE_CONCURRENCY = 8;++export function patchRequiresPerFileCommit(patch: GraphPatchPayload, bulkDeletes = false): boolean {+ if (bulkDeletes) return false;return patch.ops.some(op => op.type === 'DeleteNode' || op.type === 'DeleteEdge');}+/**+ * The first release whose bulk route applies DeleteNode/DeleteEdge in the+ * patch's own workspace, by indexed sweep and primary key (Ix-memory #211,+ * #237): checked live on that release before delete-bearing patches moved there.+ */+export const BULK_DELETES_SINCE = '1.0.32';++/**+ * Whether the backend's bulk route can take delete-bearing patches. Decided by+ * the release the backend reports; one that reports none (older than+ * Ix-memory#157, or not built by the release pipeline) keeps the per-file+ * route. `IX_BULK_DELETES=1` or `=0` overrides for such a build.+ */+export function backendAppliesBulkDeletes(+ health: { release_version?: unknown } | null | undefined,+ env: NodeJS.ProcessEnv = process.env,+): boolean {+ const forced = env.IX_BULK_DELETES;+ if (forced === '1') return true;+ if (forced === '0') return false;+ const release = health?.release_version;+ if (typeof release !== 'string') return false;+ const parse = (v: string): number[] | null => {+ const m = /^v?(\d+)\.(\d+)\.(\d+)/.exec(v.trim());+ return m ? [Number(m[1]), Number(m[2]), Number(m[3])] : null;+ };+ const have = parse(release);+ const need = parse(BULK_DELETES_SINCE)!;+ if (have === null) return false;+ for (let i = 0; i < 3; i++) if (have[i] !== need[i]) return have[i] > need[i];+ return true;+}+export function planDeletedFileRecovery(projectRoot: string,filePaths: string[],@@ -1737,8 +1807,14 @@ export async function ingestFiles(// format has changed in a way that invalidates existing node IDs (e.g. the// absolute→relative source_uri migration, or folding workspace_id into ids).// CLIENT_EXPECTED_SCHEMA_VERSION is shared with doctor/upgrade (backend-status).+ // Read from the same health answer. Unknown (the check failed) keeps the+ // per-file route for delete-bearing patches.+ let bulkDeletes = false;+ const reconcileLookups = createLimiter(RECONCILE_CONCURRENCY);try {const health = await readBackendHealth(client);+ bulkDeletes = backendAppliesBulkDeletes(health);+ if (debug) process.stderr.write(`\n [commit] delete-bearing patches via ${bulkDeletes ? 'bulk' : '/v1/patch'}\n`);const serverVersion = (health as any)?.schema_version;if (typeof serverVersion === 'number' && serverVersion !== CLIENT_EXPECTED_SCHEMA_VERSION) {process.stderr.write(@@ -2421,7 +2497,7 @@ export async function ingestFiles(bulkChunk = [];};for (const item of preparedPatches) {- if (patchRequiresPerFileCommit(item.patch)) {+ if (patchRequiresPerFileCommit(item.patch, bulkDeletes)) {flushBulkChunk();chunks.push([item]);} else {@@ -2692,11 +2768,11 @@ export async function ingestFiles(// one extra doomed request, the same price the per-chunk bulk already// pays and deliberately never skips.//- // DELETIONS are excluded. `runChunk` routes any patch with a- // DeleteNode/DeleteEdge op to its own single-item `client.commitPatch`,- // and `ingest-reconcile.test.ts` pins that -- the bulk endpoint is not- // trusted to apply delete ops. They fall through to the passes below,- // which commit them one at a time as their routing requires.+ // DELETIONS are excluded on a backend whose bulk route is not known to+ // apply delete ops (`backendAppliesBulkDeletes`): `runChunk` routes+ // those to their own single-item `client.commitPatch`. They fall+ // through to the passes below, which commit them one at a time as their+ // routing requires. On a current backend they ride the bulk like the rest.//// CHUNKED at `COMMIT_HTTP_MAX_FILES`, not skipped above it. Skipping// meant that lowering `IX_COMMIT_HTTP_MAX_FILES` -- which #560's@@ -2709,7 +2785,7 @@ export async function ingestFiles(// confirmed would be counted as a commit error -- suppressing the mtime// baseline and reporting files as missing from a graph that holds them,// which is the over-reporting `replay` exists to prevent (Ix#495).- const needsPerFile = (item: PreparedPatch): boolean => patchRequiresPerFileCommit(item.patch);+ const needsPerFile = (item: PreparedPatch): boolean => patchRequiresPerFileCommit(item.patch, bulkDeletes);const stillToSend: PreparedPatch[] = items.filter(needsPerFile);let probeQueue = items.filter(item => !needsPerFile(item));@@ -2899,7 +2975,7 @@ export async function ingestFiles(): Promise<void> =>commitItemsSerially(items, ms => { commitMsPerChunk[ci] += ms; }, bulkError, fallbackOpts);- const hasDeletion = chunk.some(item => patchRequiresPerFileCommit(item.patch));+ const hasDeletion = chunk.some(item => patchRequiresPerFileCommit(item.patch, bulkDeletes));if (hasDeletion) {await commitIndividually(chunk);@@ -3085,6 +3161,43 @@ export async function ingestFiles(return commitMsPerChunk.reduce((a, b) => a + b, 0) + drainMs;};+ /**+ * Build each file's patch and, for a file the backend had before, reconcile+ * what it removed. Reconcile runs for several files at once: awaited one+ * file at a time it was a serial chain of entity lookups per batch. The+ * patches come back in file order, without the ones that failed to build.+ */+ const buildFilePatches = async (+ files: ParsedFile[],+ edgesByFile: Map<string, any[]>,+ ): Promise<PreparedPatch[]> => {+ const built = await mapLimit(files, RECONCILE_CONCURRENCY, async ({ parsed: p, hash, previousHash }, j) => {+ try {+ const fileWorkspace = fileWorkspaceId(p.filePath);+ let patch = buildPatchFn!(p, hash, fileWorkspace, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));+ if (previousHash) {+ patch = await reconcileRemovedEntities(+ client,+ patch,+ sourcePatchIdCandidates(p.filePath, previousHash, fileWorkspace),+ undefined,+ mapMode,+ reconcileLookups,+ );+ }+ if (mapMode) patch = stripMapModeOps(patch);+ // source.uri (workspace-relative) and source.workspaceId are set+ // inside buildPatch; the backend stores both as opaque attributes.+ return makePreparedPatch(patch, j + 1, p.filePath, previousHash !== undefined || isForced(p.filePath));+ } catch (err) {+ parseErrors++;+ process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);+ return null;+ }+ });+ return built.filter((pp): pp is PreparedPatch => pp !== null);+ };+/** Resolve edges within a batch, build patches, and commit in sub-chunks. */const flushBatch = async (batch: ParsedFile[]): Promise<void> => {if (batch.length === 0) return;@@ -3114,30 +3227,7 @@ export async function ingestFiles(setCurrentWork(`build commit payloads ending ${nodePath.basename(batch[batch.length - 1].filePath)}`);const buildStart = performance.now();- const preparedPatches: PreparedPatch[] = [];- for (let j = 0; j < batch.length; j++) {- const { parsed: p, hash, previousHash } = batch[j];- try {- const fileWorkspace = fileWorkspaceId(p.filePath);- let patch = buildPatchFn!(p, hash, fileWorkspace, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));- if (previousHash) {- patch = await reconcileRemovedEntities(- client,- patch,- sourcePatchIdCandidates(p.filePath, previousHash, fileWorkspace),- undefined,- mapMode,- );- }- if (mapMode) patch = stripMapModeOps(patch);- // source.uri (workspace-relative) and source.workspaceId are set- // inside buildPatch; the backend stores both as opaque attributes.- preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath, previousHash !== undefined || isForced(p.filePath)));- } catch (err) {- parseErrors++;- process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);- }- }+ const preparedPatches = await buildFilePatches(batch, batchEdgesByFile);buildPatchMs += Math.round(performance.now() - buildStart);for (const pp of preparedPatches) {for (const op of pp.patch.ops) {@@ -3193,29 +3283,7 @@ export async function ingestFiles(}setCurrentWork(`build commit payloads ending ${nodePath.basename(allParsed[allParsed.length - 1].filePath)}`);const buildStart = performance.now();- const preparedPatches: PreparedPatch[] = [];- for (let j = 0; j < allParsed.length; j++) {- const { parsed: p, hash, previousHash } = allParsed[j];- try {- const fileWorkspace = fileWorkspaceId(p.filePath);- let patch = buildPatchFn!(p, hash, fileWorkspace, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));- if (previousHash) {- patch = await reconcileRemovedEntities(- client,- patch,- sourcePatchIdCandidates(p.filePath, previousHash, fileWorkspace),- undefined,- mapMode,- );- }- if (mapMode) patch = stripMapModeOps(patch);- // source.uri and source.workspaceId are set inside buildPatch (see flushBatch).- preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath, previousHash !== undefined || isForced(p.filePath)));- } catch (err) {- parseErrors++;- process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);- }- }+ const preparedPatches = await buildFilePatches(allParsed, edgesByFile);buildPatchMs += Math.round(performance.now() - buildStart);for (const pp of preparedPatches) {for (const op of pp.patch.ops) {@@ -3586,6 +3654,7 @@ export async function ingestFiles(sourcePatchIdCandidates(relFilePath, previousHash, fileWorkspace),dependentSourceUris,mapMode,+ reconcileLookups,);if (mapMode) patch = stripMapModeOps(patch);deletedPatches.push(makePreparedPatch(patch, i + 1, relFilePath, true));