Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): resume a partly-committed bulk instead of degrading to per-file (#515)

KageBinaryd349a22cbe82
diff --git a/ix-cli/src/cli/__tests__/ingest-retry.test.ts b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
index 06b8ab60..7531a318 100644
--- a/ix-cli/src/cli/__tests__/ingest-retry.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
@@ -2,8 +2,10 @@ import { describe, expect, it, vi } from 'vitest';
import {
commitBulkWithPayloadSplit,
isAbortError,
+ isBulkPartiallyCommittedError,
isPayloadTooLargeError,
isRetryableCommitConflict,
+ parseBulkCommittedPatchIds,
} from '../commands/ingest.js';
describe('isAbortError', () => {
@@ -117,3 +119,126 @@ describe('commitBulkWithPayloadSplit', () => {
expect(commitIndividually).toHaveBeenCalledWith([1, 2, 3], error);
});
});
+
+// The body the server actually sends, as the client stringifies it:
+// `throw new Error(`${status}: ${text}`)`.
+const partialBody = (ids: string[], expected: number) =>
+ new Error(
+ `409: ${JSON.stringify({
+ error: 'conflict',
+ message: `bulk request is partially committed (${ids.length}/${expected} patch IDs)`,
+ committed_patch_ids: ids,
+ committed_count: ids.length,
+ expected_count: expected,
+ })}`
+ );
+
+const item = (id: string) => ({ patch: { patchId: id } });
+
+describe('partly-committed bulk groups', () => {
+ it('recognises the 409 and reads the ids that landed', () => {
+ const err = partialBody(['p1', 'p2'], 5);
+ expect(isBulkPartiallyCommittedError(err)).toBe(true);
+ expect(parseBulkCommittedPatchIds(err)).toEqual(new Set(['p1', 'p2']));
+ });
+
+ it('does not mistake an ordinary conflict for a partial commit', () => {
+ expect(isBulkPartiallyCommittedError(new Error('409: {"error":"conflict"}'))).toBe(false);
+ });
+
+ // Undefined must mean "fall back", never "nothing landed" — an empty set
+ // would re-send patches the server already has and fail the same way.
+ it('returns undefined when the backend does not name the ids', () => {
+ const old = new Error('409: {"error":"conflict","message":"bulk request is partially committed (6/498 patch IDs)"}');
+ expect(isBulkPartiallyCommittedError(old)).toBe(true);
+ expect(parseBulkCommittedPatchIds(old)).toBeUndefined();
+ });
+
+ it('returns undefined for a body it cannot trust', () => {
+ expect(parseBulkCommittedPatchIds(new Error('409: not json at all'))).toBeUndefined();
+ expect(parseBulkCommittedPatchIds(new Error('409: {"committed_patch_ids":"p1"}'))).toBeUndefined();
+ expect(parseBulkCommittedPatchIds(new Error('409: {"committed_patch_ids":["p1",7]}'))).toBeUndefined();
+ });
+
+ it('replays what landed and re-bulks only what is missing', async () => {
+ const items = ['p1', 'p2', 'p3', 'p4', 'p5'].map(item);
+ const bulkCalls: string[][] = [];
+ const committed: string[] = [];
+ const commitIndividually =
+ vi.fn(async (_batch: ReturnType<typeof item>[], _error: unknown) => {});
+
+ await commitBulkWithPayloadSplit(items, {
+ commitBulk: async batch => {
+ bulkCalls.push(batch.map(b => b.patch.patchId));
+ if (batch.length === items.length) throw partialBody(['p1', 'p2'], items.length);
+ return batch.length;
+ },
+ onBulkCommitted: batch => committed.push(...batch.map(b => b.patch.patchId)),
+ commitIndividually,
+ patchIdOf: b => b.patch.patchId,
+ });
+
+ // The retry carries only the three that did not land, which is a different
+ // id set and therefore a different bulk group on the server.
+ expect(bulkCalls).toEqual([
+ ['p1', 'p2', 'p3', 'p4', 'p5'],
+ ['p3', 'p4', 'p5'],
+ ]);
+ expect(committed).toEqual(['p3', 'p4', 'p5']);
+ // The two that landed are replayed individually — idempotent no-ops that
+ // return their original revision, so counters and baseline stay right.
+ expect(commitIndividually).toHaveBeenCalledOnce();
+ const [replayed, replayError] = commitIndividually.mock.calls[0];
+ expect(replayed.map(b => b.patch.patchId)).toEqual(['p1', 'p2']);
+ // No bulk error passed for the replay: it is not a failure being reported.
+ expect(replayError).toBeUndefined();
+ });
+
+ it('falls back whole when the backend named no ids', async () => {
+ const items = ['p1', 'p2'].map(item);
+ const error = new Error('409: {"error":"conflict","message":"bulk request is partially committed (1/2 patch IDs)"}');
+ const commitIndividually = vi.fn(async () => {});
+
+ await commitBulkWithPayloadSplit(items, {
+ commitBulk: async () => { throw error; },
+ onBulkCommitted: vi.fn(),
+ commitIndividually,
+ patchIdOf: b => b.patch.patchId,
+ });
+
+ expect(commitIndividually).toHaveBeenCalledOnce();
+ expect(commitIndividually).toHaveBeenCalledWith(items, error);
+ });
+
+ // Guards the recursion: re-bulking a set identical to the one just rejected
+ // would land on the same group and repeat this call forever.
+ it('falls back whole when every id is claimed as landed', async () => {
+ const items = ['p1', 'p2'].map(item);
+ const error = partialBody(['p1', 'p2'], 2);
+ const commitIndividually = vi.fn(async () => {});
+ const commitBulk = vi.fn(async () => { throw error; });
+
+ await commitBulkWithPayloadSplit(items, {
+ commitBulk,
+ onBulkCommitted: vi.fn(),
+ commitIndividually,
+ patchIdOf: b => b.patch.patchId,
+ });
+
+ expect(commitBulk).toHaveBeenCalledOnce();
+ expect(commitIndividually).toHaveBeenCalledWith(items, error);
+ });
+
+ it('keeps the old whole-batch fallback when the caller cannot identify patches', async () => {
+ const error = partialBody(['p1'], 2);
+ const commitIndividually = vi.fn(async () => {});
+
+ await commitBulkWithPayloadSplit([1, 2], {
+ commitBulk: async () => { throw error; },
+ onBulkCommitted: vi.fn(),
+ commitIndividually,
+ });
+
+ expect(commitIndividually).toHaveBeenCalledWith([1, 2], error);
+ });
+});
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 2549e3b1..8b2b29d7 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -515,12 +515,59 @@ const COMMIT_CONFLICT_RETRY_PATTERNS = [
'timeout waiting to lock key',
'error: 1200',
// Transport-level failures (k8s ingress reset, socket drop under load).
- // Safe to retry because the server never received / committed the payload.
+ //
+ // These are retried because the request usually never reached the server --
+ // but "usually" is the whole of Ix#495. A bulk commit is chunked and writes
+ // its idempotency keys as each chunk lands, so a socket that drops after the
+ // server started work leaves part of the group committed. Retrying then
+ // re-sends a set the server has partly seen, and because the bulk group id is
+ // a hash of the patch ids, the retry reproduces the same group and is refused
+ // with a 409 that no further retry can clear.
+ //
+ // The retry stays (it is right far more often than not); what changed is that
+ // the resulting 409 is now recoverable — see isBulkPartiallyCommittedError.
'fetch failed',
'econnreset',
'econnrefused',
];
+/**
+ * A bulk commit the server accepted only in part.
+ *
+ * Re-sending the same patches cannot fix this: the group id is derived from the
+ * patch ids, so an identical retry lands on the identical rejected group. The
+ * way through is to commit only what is missing, which the backend tells us by
+ * naming the ids that landed (`committed_patch_ids`).
+ *
+ * Matched on the message because the client throws `${status}: ${body}`, so the
+ * JSON body is the tail of the error string.
+ */
+export function isBulkPartiallyCommittedError(err: unknown): boolean {
+ return String(err).toLowerCase().includes('partially committed');
+}
+
+/**
+ * The patch ids the server says already landed, or undefined when it did not
+ * say — a backend older than this field, or a body we cannot parse. Undefined
+ * means "fall back", never "nothing landed": treating an unparseable body as an
+ * empty set would re-send patches that are already committed.
+ */
+export function parseBulkCommittedPatchIds(err: unknown): Set<string> | undefined {
+ const text = String(err);
+ const start = text.indexOf('{');
+ const end = text.lastIndexOf('}');
+ if (start === -1 || end <= start) return undefined;
+ try {
+ const body = JSON.parse(text.slice(start, end + 1)) as { committed_patch_ids?: unknown };
+ const ids = body.committed_patch_ids;
+ if (!Array.isArray(ids)) return undefined;
+ const parsed = ids.filter((id): id is string => typeof id === 'string');
+ return parsed.length === ids.length ? new Set(parsed) : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
// An abort from the shared wall-clock deadline (or a per-request timeout) must
// NOT be retried — the whole point of the deadline is to stop work. Retrying
// here would defeat it and add 6 more backed-off requests to an already-late
@@ -575,6 +622,9 @@ export async function commitBulkWithPayloadSplit<T, R>(
onBulkCommitted: (batch: T[], result: R) => void;
commitIndividually: (batch: T[], error: unknown) => Promise<void>;
onSplit?: (batch: T[], error: unknown) => void;
+ /** Patch id of an item, so a partly-committed group can be resumed. */
+ patchIdOf?: (item: T) => string | undefined;
+ onPartialBulk?: (landed: T[], missing: T[], error: unknown) => void;
},
): Promise<void> {
if (items.length === 0) return;
@@ -583,15 +633,37 @@ export async function commitBulkWithPayloadSplit<T, R>(
try {
result = await handlers.commitBulk(items);
} catch (err) {
- if (!isPayloadTooLargeError(err) || items.length === 1) {
- await handlers.commitIndividually(items, err);
+ if (isPayloadTooLargeError(err) && items.length > 1) {
+ handlers.onSplit?.(items, err);
+ const midpoint = Math.ceil(items.length / 2);
+ await commitBulkWithPayloadSplit(items.slice(0, midpoint), handlers);
+ await commitBulkWithPayloadSplit(items.slice(midpoint), handlers);
return;
}
- handlers.onSplit?.(items, err);
- const midpoint = Math.ceil(items.length / 2);
- await commitBulkWithPayloadSplit(items.slice(0, midpoint), handlers);
- await commitBulkWithPayloadSplit(items.slice(midpoint), handlers);
+ // Part of this group landed. Split it by what the server says it already
+ // has: replay the landed ones individually (idempotent no-ops that return
+ // their original revision, so the counters and the mtime baseline stay
+ // truthful) and re-bulk the rest, which is a different id set and so a
+ // different group. Without this the whole batch degrades to the per-file
+ // path, which is serialized — hundreds of sequential commits, the several
+ // minutes reported in Ix#495.
+ const landedIds = isBulkPartiallyCommittedError(err) ? parseBulkCommittedPatchIds(err) : undefined;
+ if (landedIds !== undefined && handlers.patchIdOf) {
+ const idOf = handlers.patchIdOf;
+ const landed = items.filter(item => { const id = idOf(item); return id !== undefined && landedIds.has(id); });
+ const missing = items.filter(item => { const id = idOf(item); return id === undefined || !landedIds.has(id); });
+ // Only worth it when the split is real. If nothing (or everything) is
+ // claimed as landed, re-bulking `missing` would repeat this exact call.
+ if (landed.length > 0 && missing.length > 0) {
+ handlers.onPartialBulk?.(landed, missing, err);
+ await handlers.commitIndividually(landed, undefined);
+ await commitBulkWithPayloadSplit(missing, handlers);
+ return;
+ }
+ }
+
+ await handlers.commitIndividually(items, err);
return;
}
@@ -1389,6 +1461,13 @@ export async function ingestFiles(
try {
for (const item of items) {
try {
+ // Move the label per file. It used to be set once, before the
+ // bulk attempt, and never again -- so a fallback of several
+ // hundred serialized commits sat on `commit 1-498 of 498` with
+ // only the elapsed timer moving, which reads as a hang (Ix#495).
+ setCurrentWork(
+ `commit ${item.fileNumber} of ${totalFiles} ${nodePath.basename(item.filePath)} (per-file)`
+ );
const commitStart = performance.now();
const result = await retryOnConflict(() => client.commitPatch(item.patch), COMMIT_CONFLICT_RETRIES);
const chunkMs = Math.round(performance.now() - commitStart);
@@ -1444,6 +1523,13 @@ export async function ingestFiles(
for (const item of items) opts?.onCommitted?.(item, result.rev);
},
commitIndividually,
+ patchIdOf: item => item.patch.patchId,
+ onPartialBulk: (landed, missing, err) => {
+ if (!debug) return;
+ process.stderr.write(
+ `\n [bulk partly committed, resuming] ${landed.length} already landed, re-sending ${missing.length}: ${err}\n`
+ );
+ },
onSplit: (items, err) => {
if (!debug) return;
const first = items[0];