Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(ingest): bound bulk requests by ops; a run remembers a reduced size
KageBinary9e5d377bc62e
diff --git a/ix-cli/src/cli/__tests__/helpers/fake-backend.ts b/ix-cli/src/cli/__tests__/helpers/fake-backend.tsindex 75421a4d..bd804241 100644--- a/ix-cli/src/cli/__tests__/helpers/fake-backend.ts+++ b/ix-cli/src/cli/__tests__/helpers/fake-backend.ts@@ -29,6 +29,10 @@ export class FakeBackend {poison: string[] = [];/** Fail every commit, the Ix#560 shape. */refuseEverything = false;+ /** Answer a bulk of more than this many ops (over two or more patches) with a 413. */+ maxBulkOps: number | undefined = undefined;+ /** Put `suggestedMaxPatches` in that 413, as the bulk writer's estimate does. */+ suggestMaxPatches = false;/** Refuse re-sends of patches a 409 already confirmed. */refuseReplays = false;/**@@ -381,6 +385,20 @@ export class FakeBackend {this.aborter.abort();}+ if (path === "/v1/patches/bulk" && this.maxBulkOps !== undefined && patches.length > 1) {+ const ops = patches.reduce((sum, p) => sum + (p.ops?.length ?? 0), 0);+ if (ops > this.maxBulkOps) {+ // The bulk writer's estimate refusal (Ix-memory#232): a 413 before+ // any write, with the patch count that would fit when asked to say.+ const perPatch = ops / patches.length;+ const suggested = Math.max(1, Math.floor(this.maxBulkOps / perPatch));+ return send(413, {+ error: "payload_too_large",+ message: `bulk commit of ${patches.length} patches is over the transaction budget`,+ ...(this.suggestMaxPatches ? { suggestedMaxPatches: suggested } : {}),+ });+ }+ }if (path === "/v1/patch" && this.refuseReplays) {return send(500, { error: "500: already committed" });}diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.tsindex 3ebb83d7..76521c92 100644--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts@@ -1195,6 +1195,55 @@ describe("ingestFiles against a fake backend", () => {});});+ describe("bulk requests bounded by size (IN-05)", () => {+ // 1,100 files: three parse batches, so what the first batch learns has two+ // more batches to hold for. The backend refuses a bulk over 300 ops.+ const tooLarge = () => backend.requests.filter(r => r.path === "/v1/patches/bulk" && r.code === 413).length;+ const saved = process.env.IX_COMMIT_MAX_OPS;+ afterEach(() => {+ if (saved === undefined) delete process.env.IX_COMMIT_MAX_OPS;+ else process.env.IX_COMMIT_MAX_OPS = saved;+ });++ it("learns the size from the first 413 that suggests one, and is refused no more that run", async () => {+ fixture(1100);+ backend.maxBulkOps = 300;+ backend.suggestMaxPatches = true;++ const summary = await run();++ expect(tooLarge()).toBe(1);+ expect(summary.commitErrors).toBe(0);+ expect(summary.patchesApplied).toBe(1100);+ expect(backend.singleCount, "no per-file fallback").toBe(0);+ });++ it("halves by ops on a 413 that suggests nothing, and later batches start at the learned size", async () => {+ fixture(1100);+ backend.maxBulkOps = 300;++ const summary = await run();++ // A few halvings in the first batch only; bisecting each batch afresh+ // was three times as many.+ expect(tooLarge()).toBeGreaterThan(0);+ expect(tooLarge()).toBeLessThanOrEqual(4);+ expect(summary.commitErrors).toBe(0);+ expect(summary.patchesApplied).toBe(1100);+ });++ it("is never refused when IX_COMMIT_MAX_OPS keeps every request under the cap", async () => {+ fixture(1100);+ backend.maxBulkOps = 300;+ process.env.IX_COMMIT_MAX_OPS = "300";++ const summary = await run();++ expect(tooLarge()).toBe(0);+ expect(summary.patchesApplied).toBe(1100);+ });+ });+// ── Which workspace a path belongs to ─────────────────────────────────//// `ix ingest <path>` used to treat the path as its own workspace root: `ixdiff --git a/ix-cli/src/cli/__tests__/ingest-retry.test.ts b/ix-cli/src/cli/__tests__/ingest-retry.test.tsindex ea9e28c1..c2115c2a 100644--- a/ix-cli/src/cli/__tests__/ingest-retry.test.ts+++ b/ix-cli/src/cli/__tests__/ingest-retry.test.ts@@ -4,11 +4,13 @@ import { describe, expect, it, vi } from 'vitest';import {commitBulkWithPayloadSplit,commitFailureIndictsBackend,+ cutBulkChunks,isAbortError,isBulkPartiallyCommittedError,isPayloadTooLargeError,isRetryableCommitConflict,parseBulkCommittedPatchIds,+ parseSuggestedMaxPatches,} from '../commands/ingest.js';// Verbatim from a backend that refused a bulk save of a >1,000-file repo (Ix#516).@@ -166,6 +168,32 @@ describe('commitBulkWithPayloadSplit', () => {expect(commitIndividually).not.toHaveBeenCalled();});+ it('cuts a too-large group the way the caller says, in order and one piece at a time', async () => {+ const bulkCalls: number[][] = [];+ const committed: number[] = [];+ let inFlight = 0;+ let peak = 0;++ await commitBulkWithPayloadSplit([1, 2, 3, 4, 5, 6, 7], {+ commitBulk: async batch => {+ bulkCalls.push([...batch]);+ inFlight++;+ peak = Math.max(peak, inFlight);+ await new Promise(r => setTimeout(r, 1));+ inFlight--;+ if (batch.length > 3) throw new Error('413: {"error":"payload_too_large","suggestedMaxPatches":3}');+ return batch.length;+ },+ onBulkCommitted: batch => committed.push(...batch),+ commitIndividually: vi.fn(async () => {}),+ splitTooLarge: (batch, err) => cutBulkChunks(batch, () => 1, { maxFiles: parseSuggestedMaxPatches(err)!, maxOps: Infinity }),+ });++ expect(bulkCalls).toEqual([[1, 2, 3, 4, 5, 6, 7], [1, 2, 3], [4, 5, 6], [7]]);+ expect(committed).toEqual([1, 2, 3, 4, 5, 6, 7]);+ expect(peak).toBe(1);+ });+it('keeps the per-file path for a single rejected patch', async () => {const error = new Error('413: payload too large');const commitIndividually = vi.fn(async () => {});@@ -487,3 +515,20 @@ describe('partly-committed bulk groups', () => {expect(commitIndividually).toHaveBeenCalledWith([1, 2], error);});});++describe('bulk request bounds (IN-05)', () => {+ it('cuts by files and by ops, in order, and sends an over-size patch alone', () => {+ const ops = [10, 10, 10, 50, 10, 200, 10];+ const cut = cutBulkChunks(ops.map((n, i) => ({ i, n })), item => item.n, { maxFiles: 3, maxOps: 60 });+ expect(cut.map(c => c.map(item => item.i))).toEqual([[0, 1, 2], [3, 4], [5], [6]]);+ expect(cutBulkChunks([], () => 1, { maxFiles: 3, maxOps: 60 })).toEqual([]);+ });++ it('reads the patch count a 413 suggests, and nothing from one that has none', () => {+ expect(parseSuggestedMaxPatches(new Error('413: {"error":"payload_too_large","message":"x","suggestedMaxPatches":120}'))).toBe(120);+ expect(parseSuggestedMaxPatches(new Error('413: {"error":"payload_too_large"}'))).toBeUndefined();+ expect(parseSuggestedMaxPatches(new Error('413: {"suggestedMaxPatches":0}'))).toBeUndefined();+ expect(parseSuggestedMaxPatches(ARANGO_TRANSACTION_LIMIT_ERROR)).toBeUndefined();+ expect(parseSuggestedMaxPatches(new Error('413: request rejected'))).toBeUndefined();+ });+});diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.tsindex bc04dc3e..233457b0 100644--- a/ix-cli/src/cli/commands/ingest.ts+++ b/ix-cli/src/cli/commands/ingest.ts@@ -738,6 +738,53 @@ export function isPayloadTooLargeError(err: unknown): boolean {return PAYLOAD_TOO_LARGE_PATTERNS.some(pattern => message.includes(pattern));}+/**+ * What a 413 from the bulk writer's own estimate says one request should hold+ * (Ix-memory#232: `suggestedMaxPatches` in the body). Absent from a 413 that+ * ArangoDB raised mid-write, or a proxy's.+ */+export function parseSuggestedMaxPatches(err: unknown): number | undefined {+ const text = String(err);+ const start = text.indexOf('{');+ const end = text.lastIndexOf('}');+ if (start === -1 || end <= start) return undefined;+ try {+ const n = (JSON.parse(text.slice(start, end + 1)) as { suggestedMaxPatches?: unknown }).suggestedMaxPatches;+ return typeof n === 'number' && Number.isInteger(n) && n >= 1 ? n : undefined;+ } catch {+ return undefined;+ }+}++/** Per-request bounds on a bulk commit. A run lowers them when a commit is too large. */+export interface BulkLimits {+ maxFiles: number;+ /** Ops across the request's patches: the backend's transaction grows with them. */+ maxOps: number;+}++/**+ * Cut `items` into bulk requests in order, each within `limits`. An item over+ * `maxOps` on its own still goes, alone: one patch cannot be split.+ */+export function cutBulkChunks<T>(items: T[], opsOf: (item: T) => number, limits: BulkLimits): T[][] {+ const chunks: T[][] = [];+ let current: T[] = [];+ let ops = 0;+ for (const item of items) {+ const n = opsOf(item);+ if (current.length > 0 && (current.length >= limits.maxFiles || ops + n > limits.maxOps)) {+ chunks.push(current);+ current = [];+ ops = 0;+ }+ current.push(item);+ ops += n;+ }+ if (current.length > 0) chunks.push(current);+ return chunks;+}+export async function commitBulkWithPayloadSplit<T, R>(items: T[],handlers: {@@ -745,6 +792,15 @@ export async function commitBulkWithPayloadSplit<T, R>(onBulkCommitted: (batch: T[], result: R) => void;commitIndividually: (batch: T[], error: unknown, opts?: { replay?: boolean }) => Promise<void>;onSplit?: (batch: T[], error: unknown) => void;+ /**+ * How to cut a group the backend found too large, when the caller knows+ * better than halving: what the 413 suggested, or a size the run has+ * learned. Pieces go out in order, one at a time. Fewer than two pieces,+ * or absent, and the group is bisected.+ */+ splitTooLarge?: (batch: T[], error: unknown) => T[][] | undefined;+ /** Cut what is left of a split group to the bounds as they stand now. */+ recut?: (batch: T[]) => T[][];/** Patch id of an item, so a partly-committed group can be resumed. */patchIdOf?: (item: T) => string | undefined;onPartialBulk?: (landed: T[], missing: T[], error: unknown) => void;@@ -796,6 +852,19 @@ export async function commitBulkWithPayloadSplit<T, R>(} catch (err) {if (isPayloadTooLargeError(err) && items.length > 1) {handlers.onSplit?.(items, err);+ let pieces = handlers.splitTooLarge?.(items, err);+ if (pieces && pieces.length > 1) {+ // The rest is re-cut after each piece: a piece that is refused again+ // lowers the bounds, and the pieces after it should not each be+ // refused at the size it already failed at.+ while (pieces.length > 0) {+ const [head, ...tail] = pieces;+ await commitBulkWithPayloadSplit(head, handlers);+ const rest = tail.flat();+ pieces = rest.length === 0 ? [] : handlers.recut?.(rest) ?? tail;+ }+ return;+ }const midpoint = Math.ceil(items.length / 2);await commitBulkWithPayloadSplit(items.slice(0, midpoint), handlers);await commitBulkWithPayloadSplit(items.slice(midpoint), handlers);@@ -2394,6 +2463,38 @@ export async function ingestFiles(// and overlaps well with commit (~5s) + parse (~5s).const PARSE_STREAM_CHUNK = filePaths.length > 10_000 ? 500 : 500;const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 1000); // files per HTTP request to the backend+ // Ops per HTTP request. The backend's transaction grows with them, about+ // 4.3 KB a document against a 512 MB cap (~119k documents), so a 500-file+ // batch of a large repository could not fit and failed after 14-20 s.+ const COMMIT_MAX_OPS = parsePositiveIntEnv('IX_COMMIT_MAX_OPS', 60_000);+ /**+ * The bulk bounds for the rest of the run. A 413 lowers them, so later+ * batches start at a size that fits instead of failing and splitting again.+ */+ const bulkLimits: BulkLimits = { maxFiles: COMMIT_HTTP_MAX_FILES, maxOps: COMMIT_MAX_OPS };+ const opsOf = (item: { patch: GraphPatchPayload }): number => item.patch.ops.length;+ /**+ * Learn from a group the backend found too large, as an op budget: the+ * patch count its 413 suggests, at this group's ops per patch, or else+ * half the group's ops. Then cut the group to the new bounds.+ *+ * Ops, not the suggested count itself: the backend sizes its suggestion+ * from the group's average patch, and a count cut lets a run of larger+ * files through over the budget (against a backend with a 4 MB budget, a+ * count limit took 7 refusals to settle on a 316-file map, the op budget 3).+ */+ const learnBulkLimits = <T extends { patch: GraphPatchPayload }>(items: T[], err: unknown): T[][] => {+ const ops = items.reduce((sum, item) => sum + opsOf(item), 0);+ const suggested = parseSuggestedMaxPatches(err);+ const budget = suggested !== undefined && suggested < items.length+ ? Math.floor((ops * suggested) / items.length)+ : Math.floor(ops / 2);+ bulkLimits.maxOps = Math.max(1, Math.min(bulkLimits.maxOps, budget));+ if (debug) {+ process.stderr.write(`\n [bulk limits] now ${bulkLimits.maxFiles} files / ${bulkLimits.maxOps} ops per request\n`);+ }+ return cutBulkChunks(items, opsOf, bulkLimits);+ };const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 8); // parallel HTTP save requestsconst COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflicts// Re-sends of a commit that lost the base-rev race to another writer; see retryOnBaseRevRace.@@ -2407,7 +2508,7 @@ export async function ingestFiles(if (debug) {process.stderr.write(- `\n Save config: httpBatch=${COMMIT_HTTP_MAX_FILES} concurrency=${COMMIT_CONCURRENCY}\n`+ `\n Save config: httpBatch=${COMMIT_HTTP_MAX_FILES} maxOps=${COMMIT_MAX_OPS} concurrency=${COMMIT_CONCURRENCY}\n`);}@@ -2490,22 +2591,23 @@ export async function ingestFiles(const deferredByDeadline: PreparedPatch[] = [];const chunks: PreparedPatch[][] = [];- let bulkChunk: PreparedPatch[] = [];- const flushBulkChunk = (): void => {- if (bulkChunk.length === 0) return;- chunks.push(bulkChunk);- bulkChunk = [];+ let bulkRun: PreparedPatch[] = [];+ const flushBulkRun = (): void => {+ if (bulkRun.length === 0) return;+ chunks.push(...cutBulkChunks(bulkRun, opsOf, bulkLimits));+ bulkRun = [];};for (const item of preparedPatches) {if (patchRequiresPerFileCommit(item.patch, bulkDeletes)) {- flushBulkChunk();+ flushBulkRun();chunks.push([item]);} else {- bulkChunk.push(item);- if (bulkChunk.length === COMMIT_HTTP_MAX_FILES) flushBulkChunk();+ bulkRun.push(item);}}- flushBulkChunk();+ flushBulkRun();+ const isBulkChunk = (chunk: PreparedPatch[]): boolean =>+ !chunk.some(item => patchRequiresPerFileCommit(item.patch, bulkDeletes));const commitMsPerChunk = new Array<number>(chunks.length).fill(0);// The drain belongs to no chunk, but its time is still commit time --@@ -2790,7 +2892,7 @@ export async function ingestFiles(let probeQueue = items.filter(item => !needsPerFile(item));while (probeQueue.length > 1) {- const chunk = probeQueue.slice(0, COMMIT_HTTP_MAX_FILES);+ const chunk = cutBulkChunks(probeQueue, opsOf, bulkLimits)[0];const rest = probeQueue.slice(chunk.length);const bulkStart = performance.now();try {@@ -3058,6 +3160,8 @@ export async function ingestFiles(`\n [bulk partly committed, resuming] ${landed.length} already landed, re-sending ${missing.length}: ${err}\n`);},+ splitTooLarge: learnBulkLimits,+ recut: items => cutBulkChunks(items, opsOf, bulkLimits),onSplit: (items, err) => {if (!debug) return;const first = items[0];@@ -3082,9 +3186,11 @@ export async function ingestFiles(}};- await Promise.all(- Array.from({ length: Math.min(COMMIT_CONCURRENCY, chunks.length) }, () => worker())- );+ // Several bulk requests from one batch go one at a time. Each takes the+ // backend's exclusive lock, so sending them side by side bought no+ // throughput and lost base-rev races (BaseRevMismatch) to each other.+ const workers = chunks.filter(isBulkChunk).length > 1 ? 1 : Math.min(COMMIT_CONCURRENCY, chunks.length);+ await Promise.all(Array.from({ length: workers }, () => worker()));// Nothing stays abandoned while the backend is demonstrably accepting// writes. Five adjacent patches the backend rejects on their own merits