Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): delete-bearing patches go through bulk; reconcile concurrently

KageBinary4d1273d098e2
diff --git a/ix-cli/src/cli/__tests__/helpers/fake-backend.ts b/ix-cli/src/cli/__tests__/helpers/fake-backend.ts
index 1c007929..75421a4d 100644
--- a/ix-cli/src/cli/__tests__/helpers/fake-backend.ts
+++ b/ix-cli/src/cli/__tests__/helpers/fake-backend.ts
@@ -116,6 +116,12 @@ export class FakeBackend {
* uri). An older id is written again and becomes the head (BEW-03).
*/
semantics: "legacy" | "head" | undefined;
+ /**
+ * The `release_version` /v1/health reports. Unset, `head` reports a release
+ * whose bulk route applies deletes (BEW-03 and #237 are in it) and the others
+ * report none, as a pre-#157 backend does. `null` reports none in any mode.
+ */
+ releaseVersion: string | null | undefined = undefined;
private readonly nodes = new Map<string, { kind: unknown; name: unknown; live: boolean }>();
private readonly edges = new Map<string, {
src: unknown; dst: unknown; predicate: unknown; workspaceId: string; uri: string; live: boolean;
@@ -158,21 +164,18 @@ export class FakeBackend {
return this.heads.get(`${record.workspaceId}\0${record.uri}`) === patchId;
}
- private applyPatch(patch: SentPatch, sweep: boolean, bulkGroup: string | null): void {
+ private sweepEdgesOf(patch: SentPatch): void {
const workspaceId = patch.source?.workspaceId ?? "";
const uri = patch.source?.uri ?? "";
- this.rev++;
- if (sweep) {
- for (const e of this.edges.values()) {
- if (e.live && e.uri === uri && e.workspaceId === workspaceId) e.live = false;
- }
+ for (const e of this.edges.values()) {
+ if (e.live && e.uri === uri && e.workspaceId === workspaceId) e.live = false;
}
+ }
+
+ private applyDeletes(patch: SentPatch): void {
for (const op of patch.ops ?? []) {
const id = String(op.id);
- if (op.type === "UpsertNode") this.nodes.set(id, { kind: op.kind, name: op.name, live: true });
- else if (op.type === "UpsertEdge") {
- this.edges.set(id, { src: op.src, dst: op.dst, predicate: op.predicate, workspaceId, uri, live: true });
- } else if (op.type === "DeleteNode") {
+ if (op.type === "DeleteNode") {
const n = this.nodes.get(id);
if (n) n.live = false;
} else if (op.type === "DeleteEdge") {
@@ -180,6 +183,26 @@ export class FakeBackend {
if (e) e.live = false;
}
}
+ }
+
+ /**
+ * One patch's writes. A bulk request runs its phases across all of its
+ * patches -- every sweep, then every delete, then every upsert -- as
+ * BulkWriteApi `commitChunk` does, so a delete in one file's patch never
+ * removes what another patch of the same request writes.
+ */
+ private applyPatch(patch: SentPatch, bulkGroup: string | null, deletesApplied = false): void {
+ const workspaceId = patch.source?.workspaceId ?? "";
+ const uri = patch.source?.uri ?? "";
+ this.rev++;
+ if (!deletesApplied) this.applyDeletes(patch);
+ for (const op of patch.ops ?? []) {
+ const id = String(op.id);
+ if (op.type === "UpsertNode") this.nodes.set(id, { kind: op.kind, name: op.name, live: true });
+ else if (op.type === "UpsertEdge") {
+ this.edges.set(id, { src: op.src, dst: op.dst, predicate: op.predicate, workspaceId, uri, live: true });
+ }
+ }
if (patch.patchId === undefined) return;
this.stored.set(patch.patchId, {
rev: this.rev, workspaceId, uri, hash: patch.source?.sourceHash, ops: patch.ops ?? [], bulkGroup,
@@ -193,14 +216,16 @@ export class FakeBackend {
if (patch?.patchId !== undefined && this.alreadyCommitted(patch.patchId)) {
return send(200, { rev: this.stored.get(patch.patchId)!.rev, status: "Idempotent" });
}
- if (patch) this.applyPatch(patch, false, null);
+ if (patch) this.applyPatch(patch, null);
return send(200, { rev: this.rev, status: "Ok" });
}
const ids = patches.map((p) => p.patchId ?? "");
const group = createHash("sha256").update(ids.join(",")).digest("hex");
const done = ids.filter((id) => this.alreadyCommitted(id));
if (done.length === 0) {
- for (const p of patches) this.applyPatch(p, true, group);
+ for (const p of patches) this.sweepEdgesOf(p);
+ for (const p of patches) this.applyDeletes(p);
+ for (const p of patches) this.applyPatch(p, group, true);
return send(200, { rev: this.rev, applied: patches.length, status: "Ok" });
}
if (done.length !== ids.length) {
@@ -419,7 +444,10 @@ export class FakeBackend {
return;
}
- if (path === "/v1/health") return send(200, { status: "ok", version: "1.0.28" });
+ if (path === "/v1/health") {
+ const release = this.releaseVersion !== undefined ? this.releaseVersion : this.semantics === "head" ? "1.0.34" : null;
+ return send(200, { status: "ok", version: "1.0.28", ...(release ? { release_version: release } : {}) });
+ }
if (path === "/v1/source-hashes") {
if (this.failSourceHashes) return send(500, { error: "500: transaction begin timeout" });
let uris: string[] = [];
diff --git a/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts b/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts
index a3ed11d5..df47bb61 100644
--- a/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-equivalence.test.ts
@@ -264,10 +264,15 @@ describe("incremental ix map equals a fresh map (fake backend)", () => {
});
// F-04. Dropping the file's last import removes a node, the patch carries a
- // DeleteNode, and a delete-bearing patch goes to `/v1/patch`, which does not
- // sweep: the removed CALLS and IMPORTS edges stay live.
- it.fails("IN-04: removing a file's last call retires its edges", async () => {
- await runSequence("head", [edit("removeCall", "web/main.ts")]);
+ // DeleteNode, and a delete-bearing patch went to `/v1/patch`, which does not
+ // sweep: the removed CALLS and IMPORTS edges stayed live. A backend whose
+ // bulk route applies deletes now takes it there, and nothing goes per file.
+ it("IN-04: removing a file's last call retires its edges, with no /v1/patch", async () => {
+ await runSequence("head", [edit("removeCall", "web/main.ts")], {
+ check: (world) => {
+ expect(world.backend.requests.filter((r) => r.path === "/v1/patch"), "per-file commits").toEqual([]);
+ },
+ });
});
// F-03. Files that call a renamed or deleted function are unchanged, so
diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.ts
index 831d103d..3ebb83d7 100644
--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts
@@ -1161,6 +1161,40 @@ describe("ingestFiles against a fake backend", () => {
expect(backend.singleCount, "the fan-out is for failures only").toBe(0);
});
+ describe("a patch that deletes (IN-04)", () => {
+ const incremental = () =>
+ ingestFiles(repo, { format: "text", suppressOutput: true, printSummary: false });
+ const perFile = () => backend.requests.filter(r => r.path === "/v1/patch").length;
+ /** Map two files, then drop `g` from one of them: its patch carries a DeleteNode. */
+ const removeAFunction = async (): Promise<void> => {
+ backend.semantics = "head";
+ mkdirSync(join(repo, "src"), { recursive: true });
+ writeFileSync(join(repo, "src", "a.ts"), "export function f(): number { return 1; }\nexport function g(): number { return 2; }\n", "utf8");
+ writeFileSync(join(repo, "src", "b.ts"), "export const b = 1;\n", "utf8");
+ execFileSync("git", ["init", "-q"], { cwd: repo, stdio: "ignore" });
+ execFileSync("git", ["add", "-A"], { cwd: repo, stdio: "ignore" });
+ await incremental();
+ writeFileSync(join(repo, "src", "a.ts"), "export function f(): number { return 1; }\n", "utf8");
+ backend.resetRequests();
+ await incremental();
+ };
+
+ it("goes through the bulk route on a backend that applies bulk deletes", async () => {
+ backend.releaseVersion = "1.0.32";
+ await removeAFunction();
+ expect(backend.lastOps.get("src/a.ts")?.some(op => op.type === "DeleteNode"), "the patch deletes").toBe(true);
+ expect(backend.bulkCount).toBe(1);
+ expect(perFile()).toBe(0);
+ });
+
+ it("goes to /v1/patch on its own on a backend that reports no release, or an older one", async () => {
+ backend.releaseVersion = null;
+ await removeAFunction();
+ expect(backend.lastOps.get("src/a.ts")?.some(op => op.type === "DeleteNode"), "the patch deletes").toBe(true);
+ expect(perFile()).toBe(1);
+ });
+ });
+
// ── Which workspace a path belongs to ─────────────────────────────────
//
// `ix ingest <path>` used to treat the path as its own workspace root: `ix
diff --git a/ix-cli/src/cli/__tests__/ingest-reconcile.test.ts b/ix-cli/src/cli/__tests__/ingest-reconcile.test.ts
index 54b3b280..4211fca8 100644
--- a/ix-cli/src/cli/__tests__/ingest-reconcile.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-reconcile.test.ts
@@ -5,10 +5,13 @@ import * as nodePath from "node:path";
import type { GraphPatchPayload } from "../../client/types.js";
import {
+ BULK_DELETES_SINCE,
+ backendAppliesBulkDeletes,
patchRequiresPerFileCommit,
planDeletedFileRecovery,
reconcileRemovedEntities,
} from "../commands/ingest.js";
+import { createLimiter } from "../tree-walk.js";
function patchWith(ops: GraphPatchPayload["ops"]): GraphPatchPayload {
return {
@@ -29,10 +32,72 @@ function patchWith(ops: GraphPatchPayload["ops"]): GraphPatchPayload {
}
describe("reconcileRemovedEntities", () => {
- it("routes patches with deletion ops away from the bulk endpoint", () => {
+ it("routes patches with deletion ops away from the bulk endpoint, unless it applies them", () => {
expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteNode", id: "old-node" }]))).toBe(true);
expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteEdge", id: "old-edge" }]))).toBe(true);
expect(patchRequiresPerFileCommit(patchWith([{ type: "UpsertNode", id: "node" }]))).toBe(false);
+ expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteNode", id: "old-node" }]), true)).toBe(false);
+ expect(patchRequiresPerFileCommit(patchWith([{ type: "DeleteEdge", id: "old-edge" }]), true)).toBe(false);
+ });
+
+ it("trusts the bulk route with deletes from the release that applies them", () => {
+ const env = {};
+ expect(backendAppliesBulkDeletes({ release_version: BULK_DELETES_SINCE }, env)).toBe(true);
+ expect(backendAppliesBulkDeletes({ release_version: "1.0.34" }, env)).toBe(true);
+ expect(backendAppliesBulkDeletes({ release_version: "v1.1.0" }, env)).toBe(true);
+ expect(backendAppliesBulkDeletes({ release_version: "1.0.31" }, env)).toBe(false);
+ expect(backendAppliesBulkDeletes({ release_version: "0.9.99" }, env)).toBe(false);
+ // No release, or not a version: an image the release pipeline did not build.
+ expect(backendAppliesBulkDeletes({}, env)).toBe(false);
+ expect(backendAppliesBulkDeletes({ release_version: "dev" }, env)).toBe(false);
+ expect(backendAppliesBulkDeletes(null, env)).toBe(false);
+ expect(backendAppliesBulkDeletes({}, { IX_BULK_DELETES: "1" })).toBe(true);
+ expect(backendAppliesBulkDeletes({ release_version: "1.0.34" }, { IX_BULK_DELETES: "0" })).toBe(false);
+ });
+
+ it("looks up no chunk the stored patch names as one, in map mode", async () => {
+ const getPatch = vi.fn().mockResolvedValue({
+ data: {
+ ops: [
+ { type: "UpsertNode", id: "file-node", kind: "file" },
+ { type: "UpsertNode", id: "chunk-L10", kind: "chunk" },
+ { type: "UpsertNode", id: "gone-fn", kind: "function" },
+ ],
+ },
+ });
+ const entity = vi.fn().mockResolvedValue({ node: { kind: "function" }, claims: [], edges: [] });
+ const patch = patchWith([{ type: "UpsertNode", id: "file-node", kind: "file", name: "example.ts" }]);
+
+ const reconciled = await reconcileRemovedEntities({ getPatch, entity }, patch, ["prev"], undefined, true);
+
+ expect(entity.mock.calls.map(c => c[0])).toEqual(["gone-fn"]);
+ expect(reconciled.ops.filter(o => o.type === "DeleteNode").map(o => o["id"])).toEqual(["gone-fn"]);
+ });
+
+ it("runs the entity lookups within the limit, and keeps candidate order whatever finishes first", async () => {
+ const ids = Array.from({ length: 20 }, (_, i) => `n${i}`);
+ const getPatch = vi.fn().mockResolvedValue({
+ data: { ops: ids.map(id => ({ type: "UpsertNode", id, kind: "function" })) },
+ });
+ let active = 0;
+ let peak = 0;
+ const entity = vi.fn().mockImplementation(async (id: string) => {
+ active++;
+ peak = Math.max(peak, active);
+ // Later ids finish first.
+ await new Promise(r => setTimeout(r, 40 - Number(id.slice(1)) * 2));
+ active--;
+ return { node: {}, claims: [], edges: [{ id: `e-${id}`, provenance: { sourceUri: "src/caller.ts" } }] };
+ });
+ const patch = patchWith([]);
+
+ const reconciled = await reconcileRemovedEntities(
+ { getPatch, entity }, patch, ["prev"], undefined, false, createLimiter(4),
+ );
+
+ expect(peak).toBe(4);
+ expect(reconciled.ops.filter(o => o.type === "DeleteNode").map(o => o["id"])).toEqual(ids);
+ expect(reconciled.ops.filter(o => o.type === "DeleteEdge").map(o => o["id"])).toEqual(ids.map(id => `e-${id}`));
});
it("reingests surviving dependents when a deleted file returns", () => {
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index b7c7e070..bc04dc3e 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -55,6 +55,8 @@ import {
import { printJson } from '../format.js';
import { describeReplayedChanges } from '../graph-health.js';
import { createClient } from "../../client/factory.js";
+import { mapLimit } from "../../client/request-memo.js";
+import { createLimiter, type Limiter } from "../tree-walk.js";
// ---------------------------------------------------------------------------
// File discovery
// ---------------------------------------------------------------------------
@@ -301,6 +303,8 @@ export function stripMapModeOps(patch: GraphPatchPayload): GraphPatchPayload {
interface StoredPatchEntities {
nodeIds: string[];
+ /** Node kinds the stored patch wrote, when it kept its ops; empty otherwise. */
+ kinds: Map<string, string>;
}
function readStoredPatchEntities(raw: unknown): StoredPatchEntities | null {
@@ -311,12 +315,17 @@ function readStoredPatchEntities(raw: unknown): StoredPatchEntities | null {
} | undefined;
if (!data) return null;
if (Array.isArray(data.ops)) {
- const ops = data.ops as Array<{ type?: unknown; id?: unknown }>;
+ const ops = data.ops as Array<{ type?: unknown; id?: unknown; kind?: unknown }>;
+ const kinds = new Map<string, string>();
+ for (const op of ops) {
+ if (op.type === 'UpsertNode' && typeof op.id === 'string' && typeof op.kind === 'string') kinds.set(op.id, op.kind);
+ }
return {
nodeIds: ops
.filter(op => op.type === 'UpsertNode' || op.type === 'DeleteNode')
.map(op => op.id)
.filter((id): id is string => typeof id === 'string'),
+ kinds,
};
}
if (!Array.isArray(data.entityIds)) return null;
@@ -328,6 +337,7 @@ function readStoredPatchEntities(raw: unknown): StoredPatchEntities | null {
return {
nodeIds: entityIds.slice(0, nodeCount),
+ kinds: new Map(),
};
}
@@ -360,6 +370,9 @@ export async function reconcileRemovedEntities(
// command silently destroying embedding data the expensive one built. Chunks
// are not map mode's to reconcile, so in map mode they are left alone.
mapMode = false,
+ // Bounds the entity lookups. One limiter shared by every file a run
+ // reconciles keeps the whole run to its slots, however many files run at once.
+ lookupLimit: Limiter = task => task(),
): Promise<GraphPatchPayload> {
const previous = await loadStoredPatchEntities(client, previousPatchIds);
const currentNodeIds = new Set(
@@ -374,46 +387,57 @@ export async function reconcileRemovedEntities(
.map(op => op['id'])
.filter((id): id is string => typeof id === 'string'),
);
- const candidateNodeIds = previous.nodeIds.filter(id => !currentNodeIds.has(id));
+ // A chunk the stored patch names as one is skipped in map mode without a
+ // lookup; patches stored without their ops still need the lookup for it.
+ const candidateNodeIds = previous.nodeIds.filter(id =>
+ !currentNodeIds.has(id) && !(mapMode && previous.kinds.get(id) === 'chunk'));
const removedNodeIds: string[] = [];
const removedEdgeIds = new Set<string>();
const CHUNK_PREDICATES = new Set(['CONTAINS_CHUNK', 'NEXT']);
- for (const nodeId of candidateNodeIds) {
+ // The lookups run concurrently; their results are walked in candidate order,
+ // so the patch comes out the same however they complete.
+ const lookups = await Promise.all(candidateNodeIds.map(nodeId => lookupLimit(async () => {
try {
- const entity = await client.entity(nodeId);
- // The lookup is already being made for the incident edges, so the kind
- // costs nothing extra. A node whose kind cannot be read (404 — already
- // gone) falls through to the delete, which is a no-op server-side.
- if (mapMode && (entity.node as { kind?: unknown } | undefined)?.kind === 'chunk') {
- continue;
- }
- removedNodeIds.push(nodeId);
- for (const edge of entity.edges ?? []) {
- const edgeRecord = edge as {
- id?: unknown;
- provenance?: { sourceUri?: unknown; source_uri?: unknown };
- };
- const edgeId = edgeRecord.id;
- const predicate = (edge as { predicate?: unknown }).predicate;
- const isChunkEdge = typeof predicate === 'string' && CHUNK_PREDICATES.has(predicate);
- if (typeof edgeId === 'string' && !currentEdgeIds.has(edgeId) && !(mapMode && isChunkEdge)) {
- removedEdgeIds.add(edgeId);
- }
- const sourceUri = edgeRecord.provenance?.sourceUri ?? edgeRecord.provenance?.source_uri;
- if (
- dependentSourceUris &&
- typeof sourceUri === 'string' &&
- sourceUri !== patch.source.uri
- ) {
- dependentSourceUris.add(sourceUri);
- }
- }
+ return { nodeId, entity: await client.entity(nodeId) };
} catch (err) {
if (!isNotFoundError(err)) throw err;
+ return { nodeId, entity: null };
+ }
+ })));
+
+ for (const { nodeId, entity } of lookups) {
+ if (entity === null) {
// Already absent. Emitting the delete anyway keeps the patch a complete
// statement of intent and costs nothing.
removedNodeIds.push(nodeId);
+ continue;
+ }
+ // The lookup is already being made for the incident edges, so the kind
+ // costs nothing extra.
+ if (mapMode && (entity.node as { kind?: unknown } | undefined)?.kind === 'chunk') {
+ continue;
+ }
+ removedNodeIds.push(nodeId);
+ for (const edge of entity.edges ?? []) {
+ const edgeRecord = edge as {
+ id?: unknown;
+ provenance?: { sourceUri?: unknown; source_uri?: unknown };
+ };
+ const edgeId = edgeRecord.id;
+ const predicate = (edge as { predicate?: unknown }).predicate;
+ const isChunkEdge = typeof predicate === 'string' && CHUNK_PREDICATES.has(predicate);
+ if (typeof edgeId === 'string' && !currentEdgeIds.has(edgeId) && !(mapMode && isChunkEdge)) {
+ removedEdgeIds.add(edgeId);
+ }
+ const sourceUri = edgeRecord.provenance?.sourceUri ?? edgeRecord.provenance?.source_uri;
+ if (
+ dependentSourceUris &&
+ typeof sourceUri === 'string' &&
+ sourceUri !== patch.source.uri
+ ) {
+ dependentSourceUris.add(sourceUri);
+ }
}
}
@@ -436,10 +460,56 @@ export async function reconcileRemovedEntities(
};
}
-export function patchRequiresPerFileCommit(patch: GraphPatchPayload): boolean {
+/**
+ * Whether `patch` must go to `/v1/patch` on its own. Only a patch that deletes,
+ * and only against a backend whose bulk route is not known to apply deletes.
+ * `/v1/patch` does not sweep the file's old edges, so a removed call between
+ * two surviving symbols stayed live there; the bulk route sweeps them.
+ */
+/**
+ * Files reconciled at once, and entity lookups in flight across all of them:
+ * the graph walkers' cap (`EXPAND_CONCURRENCY`), for the same reason.
+ */
+const RECONCILE_CONCURRENCY = 8;
+
+export function patchRequiresPerFileCommit(patch: GraphPatchPayload, bulkDeletes = false): boolean {
+ if (bulkDeletes) return false;
return patch.ops.some(op => op.type === 'DeleteNode' || op.type === 'DeleteEdge');
}
+/**
+ * The first release whose bulk route applies DeleteNode/DeleteEdge in the
+ * patch's own workspace, by indexed sweep and primary key (Ix-memory #211,
+ * #237): checked live on that release before delete-bearing patches moved there.
+ */
+export const BULK_DELETES_SINCE = '1.0.32';
+
+/**
+ * Whether the backend's bulk route can take delete-bearing patches. Decided by
+ * the release the backend reports; one that reports none (older than
+ * Ix-memory#157, or not built by the release pipeline) keeps the per-file
+ * route. `IX_BULK_DELETES=1` or `=0` overrides for such a build.
+ */
+export function backendAppliesBulkDeletes(
+ health: { release_version?: unknown } | null | undefined,
+ env: NodeJS.ProcessEnv = process.env,
+): boolean {
+ const forced = env.IX_BULK_DELETES;
+ if (forced === '1') return true;
+ if (forced === '0') return false;
+ const release = health?.release_version;
+ if (typeof release !== 'string') return false;
+ const parse = (v: string): number[] | null => {
+ const m = /^v?(\d+)\.(\d+)\.(\d+)/.exec(v.trim());
+ return m ? [Number(m[1]), Number(m[2]), Number(m[3])] : null;
+ };
+ const have = parse(release);
+ const need = parse(BULK_DELETES_SINCE)!;
+ if (have === null) return false;
+ for (let i = 0; i < 3; i++) if (have[i] !== need[i]) return have[i] > need[i];
+ return true;
+}
+
export function planDeletedFileRecovery(
projectRoot: string,
filePaths: string[],
@@ -1737,8 +1807,14 @@ export async function ingestFiles(
// format has changed in a way that invalidates existing node IDs (e.g. the
// absolute→relative source_uri migration, or folding workspace_id into ids).
// CLIENT_EXPECTED_SCHEMA_VERSION is shared with doctor/upgrade (backend-status).
+ // Read from the same health answer. Unknown (the check failed) keeps the
+ // per-file route for delete-bearing patches.
+ let bulkDeletes = false;
+ const reconcileLookups = createLimiter(RECONCILE_CONCURRENCY);
try {
const health = await readBackendHealth(client);
+ bulkDeletes = backendAppliesBulkDeletes(health);
+ if (debug) process.stderr.write(`\n [commit] delete-bearing patches via ${bulkDeletes ? 'bulk' : '/v1/patch'}\n`);
const serverVersion = (health as any)?.schema_version;
if (typeof serverVersion === 'number' && serverVersion !== CLIENT_EXPECTED_SCHEMA_VERSION) {
process.stderr.write(
@@ -2421,7 +2497,7 @@ export async function ingestFiles(
bulkChunk = [];
};
for (const item of preparedPatches) {
- if (patchRequiresPerFileCommit(item.patch)) {
+ if (patchRequiresPerFileCommit(item.patch, bulkDeletes)) {
flushBulkChunk();
chunks.push([item]);
} else {
@@ -2692,11 +2768,11 @@ export async function ingestFiles(
// one extra doomed request, the same price the per-chunk bulk already
// pays and deliberately never skips.
//
- // DELETIONS are excluded. `runChunk` routes any patch with a
- // DeleteNode/DeleteEdge op to its own single-item `client.commitPatch`,
- // and `ingest-reconcile.test.ts` pins that -- the bulk endpoint is not
- // trusted to apply delete ops. They fall through to the passes below,
- // which commit them one at a time as their routing requires.
+ // DELETIONS are excluded on a backend whose bulk route is not known to
+ // apply delete ops (`backendAppliesBulkDeletes`): `runChunk` routes
+ // those to their own single-item `client.commitPatch`. They fall
+ // through to the passes below, which commit them one at a time as their
+ // routing requires. On a current backend they ride the bulk like the rest.
//
// CHUNKED at `COMMIT_HTTP_MAX_FILES`, not skipped above it. Skipping
// meant that lowering `IX_COMMIT_HTTP_MAX_FILES` -- which #560's
@@ -2709,7 +2785,7 @@ export async function ingestFiles(
// confirmed would be counted as a commit error -- suppressing the mtime
// baseline and reporting files as missing from a graph that holds them,
// which is the over-reporting `replay` exists to prevent (Ix#495).
- const needsPerFile = (item: PreparedPatch): boolean => patchRequiresPerFileCommit(item.patch);
+ const needsPerFile = (item: PreparedPatch): boolean => patchRequiresPerFileCommit(item.patch, bulkDeletes);
const stillToSend: PreparedPatch[] = items.filter(needsPerFile);
let probeQueue = items.filter(item => !needsPerFile(item));
@@ -2899,7 +2975,7 @@ export async function ingestFiles(
): Promise<void> =>
commitItemsSerially(items, ms => { commitMsPerChunk[ci] += ms; }, bulkError, fallbackOpts);
- const hasDeletion = chunk.some(item => patchRequiresPerFileCommit(item.patch));
+ const hasDeletion = chunk.some(item => patchRequiresPerFileCommit(item.patch, bulkDeletes));
if (hasDeletion) {
await commitIndividually(chunk);
@@ -3085,6 +3161,43 @@ export async function ingestFiles(
return commitMsPerChunk.reduce((a, b) => a + b, 0) + drainMs;
};
+ /**
+ * Build each file's patch and, for a file the backend had before, reconcile
+ * what it removed. Reconcile runs for several files at once: awaited one
+ * file at a time it was a serial chain of entity lookups per batch. The
+ * patches come back in file order, without the ones that failed to build.
+ */
+ const buildFilePatches = async (
+ files: ParsedFile[],
+ edgesByFile: Map<string, any[]>,
+ ): Promise<PreparedPatch[]> => {
+ const built = await mapLimit(files, RECONCILE_CONCURRENCY, async ({ parsed: p, hash, previousHash }, j) => {
+ try {
+ const fileWorkspace = fileWorkspaceId(p.filePath);
+ let patch = buildPatchFn!(p, hash, fileWorkspace, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));
+ if (previousHash) {
+ patch = await reconcileRemovedEntities(
+ client,
+ patch,
+ sourcePatchIdCandidates(p.filePath, previousHash, fileWorkspace),
+ undefined,
+ mapMode,
+ reconcileLookups,
+ );
+ }
+ if (mapMode) patch = stripMapModeOps(patch);
+ // source.uri (workspace-relative) and source.workspaceId are set
+ // inside buildPatch; the backend stores both as opaque attributes.
+ return makePreparedPatch(patch, j + 1, p.filePath, previousHash !== undefined || isForced(p.filePath));
+ } catch (err) {
+ parseErrors++;
+ process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);
+ return null;
+ }
+ });
+ return built.filter((pp): pp is PreparedPatch => pp !== null);
+ };
+
/** Resolve edges within a batch, build patches, and commit in sub-chunks. */
const flushBatch = async (batch: ParsedFile[]): Promise<void> => {
if (batch.length === 0) return;
@@ -3114,30 +3227,7 @@ export async function ingestFiles(
setCurrentWork(`build commit payloads ending ${nodePath.basename(batch[batch.length - 1].filePath)}`);
const buildStart = performance.now();
- const preparedPatches: PreparedPatch[] = [];
- for (let j = 0; j < batch.length; j++) {
- const { parsed: p, hash, previousHash } = batch[j];
- try {
- const fileWorkspace = fileWorkspaceId(p.filePath);
- let patch = buildPatchFn!(p, hash, fileWorkspace, batchEdgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));
- if (previousHash) {
- patch = await reconcileRemovedEntities(
- client,
- patch,
- sourcePatchIdCandidates(p.filePath, previousHash, fileWorkspace),
- undefined,
- mapMode,
- );
- }
- if (mapMode) patch = stripMapModeOps(patch);
- // source.uri (workspace-relative) and source.workspaceId are set
- // inside buildPatch; the backend stores both as opaque attributes.
- preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath, previousHash !== undefined || isForced(p.filePath)));
- } catch (err) {
- parseErrors++;
- process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);
- }
- }
+ const preparedPatches = await buildFilePatches(batch, batchEdgesByFile);
buildPatchMs += Math.round(performance.now() - buildStart);
for (const pp of preparedPatches) {
for (const op of pp.patch.ops) {
@@ -3193,29 +3283,7 @@ export async function ingestFiles(
}
setCurrentWork(`build commit payloads ending ${nodePath.basename(allParsed[allParsed.length - 1].filePath)}`);
const buildStart = performance.now();
- const preparedPatches: PreparedPatch[] = [];
- for (let j = 0; j < allParsed.length; j++) {
- const { parsed: p, hash, previousHash } = allParsed[j];
- try {
- const fileWorkspace = fileWorkspaceId(p.filePath);
- let patch = buildPatchFn!(p, hash, fileWorkspace, edgesByFile.get(p.filePath) ?? emptyEdges, previousHash, fileMultiRepo(p.filePath));
- if (previousHash) {
- patch = await reconcileRemovedEntities(
- client,
- patch,
- sourcePatchIdCandidates(p.filePath, previousHash, fileWorkspace),
- undefined,
- mapMode,
- );
- }
- if (mapMode) patch = stripMapModeOps(patch);
- // source.uri and source.workspaceId are set inside buildPatch (see flushBatch).
- preparedPatches.push(makePreparedPatch(patch, j + 1, p.filePath, previousHash !== undefined || isForced(p.filePath)));
- } catch (err) {
- parseErrors++;
- process.stderr.write(`\n [patch build error] ${p.filePath}: ${err}\n`);
- }
- }
+ const preparedPatches = await buildFilePatches(allParsed, edgesByFile);
buildPatchMs += Math.round(performance.now() - buildStart);
for (const pp of preparedPatches) {
for (const op of pp.patch.ops) {
@@ -3586,6 +3654,7 @@ export async function ingestFiles(
sourcePatchIdCandidates(relFilePath, previousHash, fileWorkspace),
dependentSourceUris,
mapMode,
+ reconcileLookups,
);
if (mapMode) patch = stripMapModeOps(patch);
deletedPatches.push(makePreparedPatch(patch, i + 1, relFilePath, true));