Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): bisect a bulk refused for exceeding Arango's transaction size (#517)

KageBinary86346da21c32
diff --git a/ix-cli/src/cli/__tests__/ingest-retry.test.ts b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
index 7531a318..69db9a53 100644
--- a/ix-cli/src/cli/__tests__/ingest-retry.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
@@ -8,6 +8,16 @@ import {
parseBulkCommittedPatchIds,
} from '../commands/ingest.js';
+// Verbatim from a backend that refused a bulk save of a >1,000-file repo (Ix#516).
+// A bulk commit runs as one exclusive Arango transaction, and Arango caps that at
+// 512MB; the driver error is relayed by ErrorHandler's catch-all arm, so it reaches
+// the CLI as a 500 body rather than the 413 the proxy limits produce.
+const ARANGO_TRANSACTION_LIMIT_ERROR = new Error(
+ '500: {"error":"internal_error","message":"Response: 500, Error: 32 - AQL: ' +
+ 'Maximal transaction size limit of 536870912 bytes is reached ' +
+ '[node #5: InsertNode] (while executing)"}',
+);
+
describe('isAbortError', () => {
it('detects AbortError and TimeoutError by name', () => {
const abort = Object.assign(new Error('The operation was aborted'), { name: 'AbortError' });
@@ -58,6 +68,10 @@ describe('isPayloadTooLargeError', () => {
expect(isPayloadTooLargeError(error)).toBe(true);
});
+ it("recognises Arango's transaction size ceiling, which arrives as a 500", () => {
+ expect(isPayloadTooLargeError(ARANGO_TRANSACTION_LIMIT_ERROR)).toBe(true);
+ });
+
it('does not mistake an unrelated error for a payload limit', () => {
expect(isPayloadTooLargeError(new Error('500: internal server error'))).toBe(false);
expect(isPayloadTooLargeError(new Error('patch 413 failed validation'))).toBe(false);
@@ -105,6 +119,34 @@ describe('commitBulkWithPayloadSplit', () => {
expect(commitIndividually).toHaveBeenCalledWith([1], error);
});
+ it('bisects an over-limit Arango transaction instead of degrading to per-file', async () => {
+ // The regression behind Ix#516: this error was unrecognised, so a whole chunk
+ // took the per-file path — 1,000 patches committed one at a time, serialized
+ // behind the fallback mutex, with the progress bar parked on the last chunk
+ // boundary. Splitting the group puts each half in its own transaction instead.
+ const bulkCalls: number[][] = [];
+ const committed: number[] = [];
+ const commitIndividually = vi.fn(async () => {});
+
+ await commitBulkWithPayloadSplit([1, 2, 3, 4], {
+ commitBulk: async batch => {
+ bulkCalls.push([...batch]);
+ if (batch.length > 2) throw ARANGO_TRANSACTION_LIMIT_ERROR;
+ return batch.length;
+ },
+ onBulkCommitted: batch => committed.push(...batch),
+ commitIndividually,
+ });
+
+ expect(bulkCalls).toEqual([
+ [1, 2, 3, 4],
+ [1, 2],
+ [3, 4],
+ ]);
+ expect(committed).toEqual([1, 2, 3, 4]);
+ expect(commitIndividually).not.toHaveBeenCalled();
+ });
+
it('keeps the existing per-file fallback for non-payload bulk failures', async () => {
const error = new Error('500: internal server error');
const commitIndividually = vi.fn(async () => {});
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 8b2b29d7..2aaf3645 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -599,11 +599,23 @@ async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Pro
}
}
+// Every entry here means the same thing operationally: this batch was refused for
+// being too big to apply as one unit, so committing fewer patches per request can
+// still succeed. That is what makes bisecting the right recovery.
const PAYLOAD_TOO_LARGE_PATTERNS = [
+ // A proxy / ingress refused the request body before the backend saw it.
'payload too large',
'request entity too large',
'content too large',
'request body too large',
+ // The backend accepted the body, then ArangoDB refused the write: a bulk commit
+ // runs as ONE exclusive transaction, and Arango caps a transaction at 512MB by
+ // default (error 32, surfaced as a 500 — not a 413):
+ // "AQL: Maximal transaction size limit of 536870912 bytes is reached"
+ // Bisecting splits the group across two transactions, so each half fits.
+ // Without this, a big-repo save fell through to the per-file path and a single
+ // 1,000-file chunk became 1,000 serialized commits — Ix#516.
+ 'maximal transaction size',
];
export function isPayloadTooLargeError(err: unknown): boolean {