Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): bound bulk requests by ops; a run remembers a reduced size

KageBinary9e5d377bc62e
diff --git a/ix-cli/src/cli/__tests__/helpers/fake-backend.ts b/ix-cli/src/cli/__tests__/helpers/fake-backend.ts
index 75421a4d..bd804241 100644
--- a/ix-cli/src/cli/__tests__/helpers/fake-backend.ts
+++ b/ix-cli/src/cli/__tests__/helpers/fake-backend.ts
@@ -29,6 +29,10 @@ export class FakeBackend {
poison: string[] = [];
/** Fail every commit, the Ix#560 shape. */
refuseEverything = false;
+ /** Answer a bulk of more than this many ops (over two or more patches) with a 413. */
+ maxBulkOps: number | undefined = undefined;
+ /** Put `suggestedMaxPatches` in that 413, as the bulk writer's estimate does. */
+ suggestMaxPatches = false;
/** Refuse re-sends of patches a 409 already confirmed. */
refuseReplays = false;
/**
@@ -381,6 +385,20 @@ export class FakeBackend {
this.aborter.abort();
}
+ if (path === "/v1/patches/bulk" && this.maxBulkOps !== undefined && patches.length > 1) {
+ const ops = patches.reduce((sum, p) => sum + (p.ops?.length ?? 0), 0);
+ if (ops > this.maxBulkOps) {
+ // The bulk writer's estimate refusal (Ix-memory#232): a 413 before
+ // any write, with the patch count that would fit when asked to say.
+ const perPatch = ops / patches.length;
+ const suggested = Math.max(1, Math.floor(this.maxBulkOps / perPatch));
+ return send(413, {
+ error: "payload_too_large",
+ message: `bulk commit of ${patches.length} patches is over the transaction budget`,
+ ...(this.suggestMaxPatches ? { suggestedMaxPatches: suggested } : {}),
+ });
+ }
+ }
if (path === "/v1/patch" && this.refuseReplays) {
return send(500, { error: "500: already committed" });
}
diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.ts
index 3ebb83d7..76521c92 100644
--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts
@@ -1195,6 +1195,55 @@ describe("ingestFiles against a fake backend", () => {
});
});
+ describe("bulk requests bounded by size (IN-05)", () => {
+ // 1,100 files: three parse batches, so what the first batch learns has two
+ // more batches to hold for. The backend refuses a bulk over 300 ops.
+ const tooLarge = () => backend.requests.filter(r => r.path === "/v1/patches/bulk" && r.code === 413).length;
+ const saved = process.env.IX_COMMIT_MAX_OPS;
+ afterEach(() => {
+ if (saved === undefined) delete process.env.IX_COMMIT_MAX_OPS;
+ else process.env.IX_COMMIT_MAX_OPS = saved;
+ });
+
+ it("learns the size from the first 413 that suggests one, and is refused no more that run", async () => {
+ fixture(1100);
+ backend.maxBulkOps = 300;
+ backend.suggestMaxPatches = true;
+
+ const summary = await run();
+
+ expect(tooLarge()).toBe(1);
+ expect(summary.commitErrors).toBe(0);
+ expect(summary.patchesApplied).toBe(1100);
+ expect(backend.singleCount, "no per-file fallback").toBe(0);
+ });
+
+ it("halves by ops on a 413 that suggests nothing, and later batches start at the learned size", async () => {
+ fixture(1100);
+ backend.maxBulkOps = 300;
+
+ const summary = await run();
+
+ // A few halvings in the first batch only; bisecting each batch afresh
+ // was three times as many.
+ expect(tooLarge()).toBeGreaterThan(0);
+ expect(tooLarge()).toBeLessThanOrEqual(4);
+ expect(summary.commitErrors).toBe(0);
+ expect(summary.patchesApplied).toBe(1100);
+ });
+
+ it("is never refused when IX_COMMIT_MAX_OPS keeps every request under the cap", async () => {
+ fixture(1100);
+ backend.maxBulkOps = 300;
+ process.env.IX_COMMIT_MAX_OPS = "300";
+
+ const summary = await run();
+
+ expect(tooLarge()).toBe(0);
+ expect(summary.patchesApplied).toBe(1100);
+ });
+ });
+
// ── Which workspace a path belongs to ─────────────────────────────────
//
// `ix ingest <path>` used to treat the path as its own workspace root: `ix
diff --git a/ix-cli/src/cli/__tests__/ingest-retry.test.ts b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
index ea9e28c1..c2115c2a 100644
--- a/ix-cli/src/cli/__tests__/ingest-retry.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
@@ -4,11 +4,13 @@ import { describe, expect, it, vi } from 'vitest';
import {
commitBulkWithPayloadSplit,
commitFailureIndictsBackend,
+ cutBulkChunks,
isAbortError,
isBulkPartiallyCommittedError,
isPayloadTooLargeError,
isRetryableCommitConflict,
parseBulkCommittedPatchIds,
+ parseSuggestedMaxPatches,
} from '../commands/ingest.js';
// Verbatim from a backend that refused a bulk save of a >1,000-file repo (Ix#516).
@@ -166,6 +168,32 @@ describe('commitBulkWithPayloadSplit', () => {
expect(commitIndividually).not.toHaveBeenCalled();
});
+ it('cuts a too-large group the way the caller says, in order and one piece at a time', async () => {
+ const bulkCalls: number[][] = [];
+ const committed: number[] = [];
+ let inFlight = 0;
+ let peak = 0;
+
+ await commitBulkWithPayloadSplit([1, 2, 3, 4, 5, 6, 7], {
+ commitBulk: async batch => {
+ bulkCalls.push([...batch]);
+ inFlight++;
+ peak = Math.max(peak, inFlight);
+ await new Promise(r => setTimeout(r, 1));
+ inFlight--;
+ if (batch.length > 3) throw new Error('413: {"error":"payload_too_large","suggestedMaxPatches":3}');
+ return batch.length;
+ },
+ onBulkCommitted: batch => committed.push(...batch),
+ commitIndividually: vi.fn(async () => {}),
+ splitTooLarge: (batch, err) => cutBulkChunks(batch, () => 1, { maxFiles: parseSuggestedMaxPatches(err)!, maxOps: Infinity }),
+ });
+
+ expect(bulkCalls).toEqual([[1, 2, 3, 4, 5, 6, 7], [1, 2, 3], [4, 5, 6], [7]]);
+ expect(committed).toEqual([1, 2, 3, 4, 5, 6, 7]);
+ expect(peak).toBe(1);
+ });
+
it('keeps the per-file path for a single rejected patch', async () => {
const error = new Error('413: payload too large');
const commitIndividually = vi.fn(async () => {});
@@ -487,3 +515,20 @@ describe('partly-committed bulk groups', () => {
expect(commitIndividually).toHaveBeenCalledWith([1, 2], error);
});
});
+
+describe('bulk request bounds (IN-05)', () => {
+ it('cuts by files and by ops, in order, and sends an over-size patch alone', () => {
+ const ops = [10, 10, 10, 50, 10, 200, 10];
+ const cut = cutBulkChunks(ops.map((n, i) => ({ i, n })), item => item.n, { maxFiles: 3, maxOps: 60 });
+ expect(cut.map(c => c.map(item => item.i))).toEqual([[0, 1, 2], [3, 4], [5], [6]]);
+ expect(cutBulkChunks([], () => 1, { maxFiles: 3, maxOps: 60 })).toEqual([]);
+ });
+
+ it('reads the patch count a 413 suggests, and nothing from one that has none', () => {
+ expect(parseSuggestedMaxPatches(new Error('413: {"error":"payload_too_large","message":"x","suggestedMaxPatches":120}'))).toBe(120);
+ expect(parseSuggestedMaxPatches(new Error('413: {"error":"payload_too_large"}'))).toBeUndefined();
+ expect(parseSuggestedMaxPatches(new Error('413: {"suggestedMaxPatches":0}'))).toBeUndefined();
+ expect(parseSuggestedMaxPatches(ARANGO_TRANSACTION_LIMIT_ERROR)).toBeUndefined();
+ expect(parseSuggestedMaxPatches(new Error('413: request rejected'))).toBeUndefined();
+ });
+});
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index bc04dc3e..233457b0 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -738,6 +738,53 @@ export function isPayloadTooLargeError(err: unknown): boolean {
return PAYLOAD_TOO_LARGE_PATTERNS.some(pattern => message.includes(pattern));
}
+/**
+ * What a 413 from the bulk writer's own estimate says one request should hold
+ * (Ix-memory#232: `suggestedMaxPatches` in the body). Absent from a 413 that
+ * ArangoDB raised mid-write, or a proxy's.
+ */
+export function parseSuggestedMaxPatches(err: unknown): number | undefined {
+ const text = String(err);
+ const start = text.indexOf('{');
+ const end = text.lastIndexOf('}');
+ if (start === -1 || end <= start) return undefined;
+ try {
+ const n = (JSON.parse(text.slice(start, end + 1)) as { suggestedMaxPatches?: unknown }).suggestedMaxPatches;
+ return typeof n === 'number' && Number.isInteger(n) && n >= 1 ? n : undefined;
+ } catch {
+ return undefined;
+ }
+}
+
+/** Per-request bounds on a bulk commit. A run lowers them when a commit is too large. */
+export interface BulkLimits {
+ maxFiles: number;
+ /** Ops across the request's patches: the backend's transaction grows with them. */
+ maxOps: number;
+}
+
+/**
+ * Cut `items` into bulk requests in order, each within `limits`. An item over
+ * `maxOps` on its own still goes, alone: one patch cannot be split.
+ */
+export function cutBulkChunks<T>(items: T[], opsOf: (item: T) => number, limits: BulkLimits): T[][] {
+ const chunks: T[][] = [];
+ let current: T[] = [];
+ let ops = 0;
+ for (const item of items) {
+ const n = opsOf(item);
+ if (current.length > 0 && (current.length >= limits.maxFiles || ops + n > limits.maxOps)) {
+ chunks.push(current);
+ current = [];
+ ops = 0;
+ }
+ current.push(item);
+ ops += n;
+ }
+ if (current.length > 0) chunks.push(current);
+ return chunks;
+}
+
export async function commitBulkWithPayloadSplit<T, R>(
items: T[],
handlers: {
@@ -745,6 +792,15 @@ export async function commitBulkWithPayloadSplit<T, R>(
onBulkCommitted: (batch: T[], result: R) => void;
commitIndividually: (batch: T[], error: unknown, opts?: { replay?: boolean }) => Promise<void>;
onSplit?: (batch: T[], error: unknown) => void;
+ /**
+ * How to cut a group the backend found too large, when the caller knows
+ * better than halving: what the 413 suggested, or a size the run has
+ * learned. Pieces go out in order, one at a time. Fewer than two pieces,
+ * or absent, and the group is bisected.
+ */
+ splitTooLarge?: (batch: T[], error: unknown) => T[][] | undefined;
+ /** Cut what is left of a split group to the bounds as they stand now. */
+ recut?: (batch: T[]) => T[][];
/** Patch id of an item, so a partly-committed group can be resumed. */
patchIdOf?: (item: T) => string | undefined;
onPartialBulk?: (landed: T[], missing: T[], error: unknown) => void;
@@ -796,6 +852,19 @@ export async function commitBulkWithPayloadSplit<T, R>(
} catch (err) {
if (isPayloadTooLargeError(err) && items.length > 1) {
handlers.onSplit?.(items, err);
+ let pieces = handlers.splitTooLarge?.(items, err);
+ if (pieces && pieces.length > 1) {
+ // The rest is re-cut after each piece: a piece that is refused again
+ // lowers the bounds, and the pieces after it should not each be
+ // refused at the size it already failed at.
+ while (pieces.length > 0) {
+ const [head, ...tail] = pieces;
+ await commitBulkWithPayloadSplit(head, handlers);
+ const rest = tail.flat();
+ pieces = rest.length === 0 ? [] : handlers.recut?.(rest) ?? tail;
+ }
+ return;
+ }
const midpoint = Math.ceil(items.length / 2);
await commitBulkWithPayloadSplit(items.slice(0, midpoint), handlers);
await commitBulkWithPayloadSplit(items.slice(midpoint), handlers);
@@ -2394,6 +2463,38 @@ export async function ingestFiles(
// and overlaps well with commit (~5s) + parse (~5s).
const PARSE_STREAM_CHUNK = filePaths.length > 10_000 ? 500 : 500;
const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 1000); // files per HTTP request to the backend
+ // Ops per HTTP request. The backend's transaction grows with them, about
+ // 4.3 KB a document against a 512 MB cap (~119k documents), so a 500-file
+ // batch of a large repository could not fit and failed after 14-20 s.
+ const COMMIT_MAX_OPS = parsePositiveIntEnv('IX_COMMIT_MAX_OPS', 60_000);
+ /**
+ * The bulk bounds for the rest of the run. A 413 lowers them, so later
+ * batches start at a size that fits instead of failing and splitting again.
+ */
+ const bulkLimits: BulkLimits = { maxFiles: COMMIT_HTTP_MAX_FILES, maxOps: COMMIT_MAX_OPS };
+ const opsOf = (item: { patch: GraphPatchPayload }): number => item.patch.ops.length;
+ /**
+ * Learn from a group the backend found too large, as an op budget: the
+ * patch count its 413 suggests, at this group's ops per patch, or else
+ * half the group's ops. Then cut the group to the new bounds.
+ *
+ * Ops, not the suggested count itself: the backend sizes its suggestion
+ * from the group's average patch, and a count cut lets a run of larger
+ * files through over the budget (against a backend with a 4 MB budget, a
+ * count limit took 7 refusals to settle on a 316-file map, the op budget 3).
+ */
+ const learnBulkLimits = <T extends { patch: GraphPatchPayload }>(items: T[], err: unknown): T[][] => {
+ const ops = items.reduce((sum, item) => sum + opsOf(item), 0);
+ const suggested = parseSuggestedMaxPatches(err);
+ const budget = suggested !== undefined && suggested < items.length
+ ? Math.floor((ops * suggested) / items.length)
+ : Math.floor(ops / 2);
+ bulkLimits.maxOps = Math.max(1, Math.min(bulkLimits.maxOps, budget));
+ if (debug) {
+ process.stderr.write(`\n [bulk limits] now ${bulkLimits.maxFiles} files / ${bulkLimits.maxOps} ops per request\n`);
+ }
+ return cutBulkChunks(items, opsOf, bulkLimits);
+ };
const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 8); // parallel HTTP save requests
const COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflicts
// Re-sends of a commit that lost the base-rev race to another writer; see retryOnBaseRevRace.
@@ -2407,7 +2508,7 @@ export async function ingestFiles(
if (debug) {
process.stderr.write(
- `\n Save config: httpBatch=${COMMIT_HTTP_MAX_FILES} concurrency=${COMMIT_CONCURRENCY}\n`
+ `\n Save config: httpBatch=${COMMIT_HTTP_MAX_FILES} maxOps=${COMMIT_MAX_OPS} concurrency=${COMMIT_CONCURRENCY}\n`
);
}
@@ -2490,22 +2591,23 @@ export async function ingestFiles(
const deferredByDeadline: PreparedPatch[] = [];
const chunks: PreparedPatch[][] = [];
- let bulkChunk: PreparedPatch[] = [];
- const flushBulkChunk = (): void => {
- if (bulkChunk.length === 0) return;
- chunks.push(bulkChunk);
- bulkChunk = [];
+ let bulkRun: PreparedPatch[] = [];
+ const flushBulkRun = (): void => {
+ if (bulkRun.length === 0) return;
+ chunks.push(...cutBulkChunks(bulkRun, opsOf, bulkLimits));
+ bulkRun = [];
};
for (const item of preparedPatches) {
if (patchRequiresPerFileCommit(item.patch, bulkDeletes)) {
- flushBulkChunk();
+ flushBulkRun();
chunks.push([item]);
} else {
- bulkChunk.push(item);
- if (bulkChunk.length === COMMIT_HTTP_MAX_FILES) flushBulkChunk();
+ bulkRun.push(item);
}
}
- flushBulkChunk();
+ flushBulkRun();
+ const isBulkChunk = (chunk: PreparedPatch[]): boolean =>
+ !chunk.some(item => patchRequiresPerFileCommit(item.patch, bulkDeletes));
const commitMsPerChunk = new Array<number>(chunks.length).fill(0);
// The drain belongs to no chunk, but its time is still commit time --
@@ -2790,7 +2892,7 @@ export async function ingestFiles(
let probeQueue = items.filter(item => !needsPerFile(item));
while (probeQueue.length > 1) {
- const chunk = probeQueue.slice(0, COMMIT_HTTP_MAX_FILES);
+ const chunk = cutBulkChunks(probeQueue, opsOf, bulkLimits)[0];
const rest = probeQueue.slice(chunk.length);
const bulkStart = performance.now();
try {
@@ -3058,6 +3160,8 @@ export async function ingestFiles(
`\n [bulk partly committed, resuming] ${landed.length} already landed, re-sending ${missing.length}: ${err}\n`
);
},
+ splitTooLarge: learnBulkLimits,
+ recut: items => cutBulkChunks(items, opsOf, bulkLimits),
onSplit: (items, err) => {
if (!debug) return;
const first = items[0];
@@ -3082,9 +3186,11 @@ export async function ingestFiles(
}
};
- await Promise.all(
- Array.from({ length: Math.min(COMMIT_CONCURRENCY, chunks.length) }, () => worker())
- );
+ // Several bulk requests from one batch go one at a time. Each takes the
+ // backend's exclusive lock, so sending them side by side bought no
+ // throughput and lost base-rev races (BaseRevMismatch) to each other.
+ const workers = chunks.filter(isBulkChunk).length > 1 ? 1 : Math.min(COMMIT_CONCURRENCY, chunks.length);
+ await Promise.all(Array.from({ length: workers }, () => worker()));
// Nothing stays abandoned while the backend is demonstrably accepting
// writes. Five adjacent patches the backend rejects on their own merits