Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): default bulk op bound under the backend's non-empty-graph budget

KageBinaryf83d01a45f6e
diff --git a/ix-cli/src/cli/__tests__/ingest-retry.test.ts b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
index c2115c2a..3fa08759 100644
--- a/ix-cli/src/cli/__tests__/ingest-retry.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
@@ -5,6 +5,7 @@ import {
commitBulkWithPayloadSplit,
commitFailureIndictsBackend,
cutBulkChunks,
+ DEFAULT_COMMIT_MAX_OPS,
isAbortError,
isBulkPartiallyCommittedError,
isPayloadTooLargeError,
@@ -517,6 +518,16 @@ describe('partly-committed bulk groups', () => {
});
describe('bulk request bounds (IN-05)', () => {
+ it('defaults under the bulk writer\'s pre-flight budget on a non-empty graph', () => {
+ // Ix-memory BulkTransactionBudget.Default: 80% of ArangoDB's 512 MB stream
+ // transaction at 4,300 bytes per upserted document, doubled once rev > 0.
+ // A default over it is refused with a 413 on the first large request of
+ // every map but a backend's first.
+ const limitBytes = Math.floor((512 * 1024 * 1024 * 8) / 10);
+ const nonEmptyGraphDocs = Math.floor(limitBytes / (4300 * 2));
+ expect(DEFAULT_COMMIT_MAX_OPS).toBeLessThanOrEqual(nonEmptyGraphDocs);
+ });
+
it('cuts by files and by ops, in order, and sends an over-size patch alone', () => {
const ops = [10, 10, 10, 50, 10, 200, 10];
const cut = cutBulkChunks(ops.map((n, i) => ({ i, n })), item => item.n, { maxFiles: 3, maxOps: 60 });
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index 86f53e12..f200e3cb 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -756,6 +756,20 @@ export function parseSuggestedMaxPatches(err: unknown): number | undefined {
}
}
+/**
+ * Ops per bulk request unless `IX_COMMIT_MAX_OPS` says otherwise. The backend's
+ * transaction grows with them, about 4.3 KB a document against ArangoDB's
+ * 512 MB cap, so a 500-file batch of a large repository could not fit and
+ * failed after 14-20 s.
+ *
+ * The bound that matters is the bulk writer's own pre-flight budget (Ix-memory
+ * `BulkTransactionBudget`, from 1.0.32): 80% of 512 MB at 4,300 bytes per
+ * upserted document, doubled once the graph holds anything, which is about
+ * 49,900 documents on every map but a backend's first. Over it, a request is
+ * refused with a 413 before any write; this default stays under it.
+ */
+export const DEFAULT_COMMIT_MAX_OPS = 45_000;
+
/** Per-request bounds on a bulk commit. A run lowers them when a commit is too large. */
export interface BulkLimits {
maxFiles: number;
@@ -2463,10 +2477,8 @@ export async function ingestFiles(
// and overlaps well with commit (~5s) + parse (~5s).
const PARSE_STREAM_CHUNK = filePaths.length > 10_000 ? 500 : 500;
const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 1000); // files per HTTP request to the backend
- // Ops per HTTP request. The backend's transaction grows with them, about
- // 4.3 KB a document against a 512 MB cap (~119k documents), so a 500-file
- // batch of a large repository could not fit and failed after 14-20 s.
- const COMMIT_MAX_OPS = parsePositiveIntEnv('IX_COMMIT_MAX_OPS', 60_000);
+ // Ops per HTTP request; see DEFAULT_COMMIT_MAX_OPS.
+ const COMMIT_MAX_OPS = parsePositiveIntEnv('IX_COMMIT_MAX_OPS', DEFAULT_COMMIT_MAX_OPS);
/**
* The bulk bounds for the rest of the run. A 413 lowers them, so later
* batches start at a size that fits instead of failing and splitting again.