Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): bisect a bulk refused for exceeding Arango's transaction size (#517)
KageBinary86346da21c32
diff --git a/ix-cli/src/cli/__tests__/ingest-retry.test.ts b/ix-cli/src/cli/__tests__/ingest-retry.test.tsindex 7531a318..69db9a53 100644--- a/ix-cli/src/cli/__tests__/ingest-retry.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-retry.test.ts@@ -8,6 +8,16 @@ import {parseBulkCommittedPatchIds,} from '../commands/ingest.js';+// Verbatim from a backend that refused a bulk save of a >1,000-file repo (Ix#516).+// A bulk commit runs as one exclusive Arango transaction, and Arango caps that at+// 512MB; the driver error is relayed by ErrorHandler's catch-all arm, so it reaches+// the CLI as a 500 body rather than the 413 the proxy limits produce.+const ARANGO_TRANSACTION_LIMIT_ERROR = new Error(+ '500: {"error":"internal_error","message":"Response: 500, Error: 32 - AQL: ' ++ 'Maximal transaction size limit of 536870912 bytes is reached ' ++ '[node #5: InsertNode] (while executing)"}',+);+describe('isAbortError', () => {it('detects AbortError and TimeoutError by name', () => {const abort = Object.assign(new Error('The operation was aborted'), { name: 'AbortError' });@@ -58,6 +68,10 @@ describe('isPayloadTooLargeError', () => {expect(isPayloadTooLargeError(error)).toBe(true);});+ it("recognises Arango's transaction size ceiling, which arrives as a 500", () => {+ expect(isPayloadTooLargeError(ARANGO_TRANSACTION_LIMIT_ERROR)).toBe(true);+ });+it('does not mistake an unrelated error for a payload limit', () => {expect(isPayloadTooLargeError(new Error('500: internal server error'))).toBe(false);expect(isPayloadTooLargeError(new Error('patch 413 failed validation'))).toBe(false);@@ -105,6 +119,34 @@ describe('commitBulkWithPayloadSplit', () => {expect(commitIndividually).toHaveBeenCalledWith([1], error);});+ it('bisects an over-limit Arango transaction instead of degrading to per-file', async () => {+ // The regression behind Ix#516: this error was unrecognised, so a whole chunk+ // took the per-file path — 1,000 patches committed one at a time, serialized+ // behind the fallback mutex, with the progress bar parked on the last chunk+ // boundary. Splitting the group puts each half in its own transaction instead.+ const bulkCalls: number[][] = [];+ const committed: number[] = [];+ const commitIndividually = vi.fn(async () => {});++ await commitBulkWithPayloadSplit([1, 2, 3, 4], {+ commitBulk: async batch => {+ bulkCalls.push([...batch]);+ if (batch.length > 2) throw ARANGO_TRANSACTION_LIMIT_ERROR;+ return batch.length;+ },+ onBulkCommitted: batch => committed.push(...batch),+ commitIndividually,+ });++ expect(bulkCalls).toEqual([+ [1, 2, 3, 4],+ [1, 2],+ [3, 4],+ ]);+ expect(committed).toEqual([1, 2, 3, 4]);+ expect(commitIndividually).not.toHaveBeenCalled();+ });+it('keeps the existing per-file fallback for non-payload bulk failures', async () => {const error = new Error('500: internal server error');const commitIndividually = vi.fn(async () => {});diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex 8b2b29d7..2aaf3645 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -599,11 +599,23 @@ async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Pro}}+// Every entry here means the same thing operationally: this batch was refused for+// being too big to apply as one unit, so committing fewer patches per request can+// still succeed. That is what makes bisecting the right recovery.const PAYLOAD_TOO_LARGE_PATTERNS = [+ // A proxy / ingress refused the request body before the backend saw it.'payload too large','request entity too large','content too large','request body too large',+ // The backend accepted the body, then ArangoDB refused the write: a bulk commit+ // runs as ONE exclusive transaction, and Arango caps a transaction at 512MB by+ // default (error 32, surfaced as a 500 — not a 413):+ // "AQL: Maximal transaction size limit of 536870912 bytes is reached"+ // Bisecting splits the group across two transactions, so each half fits.+ // Without this, a big-repo save fell through to the per-file path and a single+ // 1,000-file chunk became 1,000 serialized commits — Ix#516.+ 'maximal transaction size',];export function isPayloadTooLargeError(err: unknown): boolean {