Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): re-send commits that lost the base-rev race to another writer

KageBinary70c18d95dbbf
diff --git a/docs-site/src/content/docs/reference/configuration.md b/docs-site/src/content/docs/reference/configuration.md
index c83a07a7..0dde0ceb 100644
--- a/docs-site/src/content/docs/reference/configuration.md
+++ b/docs-site/src/content/docs/reference/configuration.md
@@ -22,6 +22,7 @@ ix config set format llm # set a value
| `IX_DEBUG=1` | Prints full stack traces on errors |
| `IX_MCP_SUBPROCESS=1` | Runs each MCP tool call in its own `ix` process |
| `IX_COMMIT_FAILURE_LIMIT=<n>` | Consecutive failed commits before `ix map` stops (default 5). `0` never stops |
+| `IX_COMMIT_BASE_REV_RETRIES=<n>` | Re-sends of a commit after another writer on the same backend moved the graph revision under it (default 8) |
### Bootstrap script variables
diff --git a/ix-cli/src/cli/__tests__/base-rev-retry.test.ts b/ix-cli/src/cli/__tests__/base-rev-retry.test.ts
new file mode 100644
index 00000000..adbb3cb5
--- /dev/null
+++ b/ix-cli/src/cli/__tests__/base-rev-retry.test.ts
@@ -0,0 +1,42 @@
+// Copyright 2026 Ix Infrastructure Inc.
+
+import { describe, expect, it } from "vitest";
+
+import { retryOnBaseRevRace } from "../commands/ingest.js";
+
+/** Answers `BaseRevMismatch` `losses` times, then `Ok`. */
+function racingCommit(losses: number) {
+ let calls = 0;
+ const fn = async () => {
+ calls++;
+ return { rev: calls, status: calls <= losses ? "BaseRevMismatch" : "Ok" };
+ };
+ return { fn, calls: () => calls };
+}
+
+describe("retryOnBaseRevRace", () => {
+ it("re-sends until the race is won and returns that result", async () => {
+ const commit = racingCommit(2);
+ const retries: number[] = [];
+ const result = await retryOnBaseRevRace(commit.fn, 5, (n) => retries.push(n));
+ expect(result.status).toBe("Ok");
+ expect(commit.calls()).toBe(3);
+ expect(retries).toEqual([1, 2]);
+ });
+
+ it("gives up after maxRetries and returns the lost race for the caller to count", async () => {
+ const commit = racingCommit(100);
+ const result = await retryOnBaseRevRace(commit.fn, 2);
+ expect(result.status).toBe("BaseRevMismatch");
+ expect(commit.calls()).toBe(3);
+ });
+
+ it("does not retry any other status", async () => {
+ for (const status of ["Ok", "Idempotent"]) {
+ let calls = 0;
+ const result = await retryOnBaseRevRace(async () => { calls++; return { status }; }, 5);
+ expect(result.status).toBe(status);
+ expect(calls).toBe(1);
+ }
+ });
+});
diff --git a/ix-cli/src/cli/__tests__/ingest-files.test.ts b/ix-cli/src/cli/__tests__/ingest-files.test.ts
index bd686a3f..9330c097 100644
--- a/ix-cli/src/cli/__tests__/ingest-files.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-files.test.ts
@@ -72,6 +72,12 @@ class FakeBackend {
* and stays right if the number of requests before it ever changes.
*/
mismatchOnDrainBulk = false;
+ /**
+ * Answer this many commits (bulk or single) `BaseRevMismatch` before taking
+ * any: what a second `ix map` committing to the same backend does to this
+ * one. A 200 that wrote nothing, like `mismatchOnDrainBulk`.
+ */
+ loseBaseRevRaces = 0;
/**
* Refuse the opening bulk and every per-file send, but ACCEPT the drain.
*
@@ -249,6 +255,10 @@ class FakeBackend {
// set the status further down and measured nothing, because no bulk in
// the test ever got there.
const isDrainBulk = path === "/v1/patches/bulk" && this.singleCount > 0;
+ if (this.loseBaseRevRaces > 0) {
+ this.loseBaseRevRaces--;
+ return send(200, { rev: this.rev, applied: 0, status: "BaseRevMismatch" });
+ }
if (this.mismatchOnDrainBulk && isDrainBulk) {
// A 200 that wrote NOTHING: the backend read the latest rev outside the
// transaction and it moved before the commit ran. `applied: 0` and the
@@ -925,6 +935,34 @@ describe("ingestFiles against a fake backend", () => {
}
});
+ it("re-sends commits that lost the base-rev race to another writer", async () => {
+ // Two `ix map` runs against one backend: each commit that lands moves the
+ // rev under the other's in-flight commit, which is answered 200
+ // BaseRevMismatch and writes nothing. Those used to be counted as failed
+ // and left for the next run -- a third of the repo missing and exit 1.
+ fixture(30);
+ backend.loseBaseRevRaces = 3;
+
+ const summary = await run();
+
+ expect(summary.commitErrors, "every lost race was re-sent and landed").toBe(0);
+ expect(summary.patchesApplied).toBe(30);
+ expect(backend.bulkCount, "the same bulk, sent until it won").toBe(4);
+ expect(backend.singleCount, "a lost race is not a reason to fan out").toBe(0);
+ });
+
+ it("still reports a race it keeps losing, after the retries", async () => {
+ fixture(12);
+ process.env.IX_COMMIT_BASE_REV_RETRIES = "2";
+ backend.loseBaseRevRaces = 1_000;
+ try {
+ const message = await runFatal().catch((err: unknown) => String(err));
+ expect(message).toMatch(/12 of 12|failed to commit/);
+ } finally {
+ delete process.env.IX_COMMIT_BASE_REV_RETRIES;
+ }
+ });
+
it("commits a healthy repo in one bulk, with no per-file fan-out", async () => {
fixture(12);
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index e47d6be9..6fa746e4 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -719,6 +719,38 @@ async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Pro
}
}
+/**
+ * Re-send a commit that lost the backend's base-rev race.
+ *
+ * `BaseRevMismatch` means the backend read the latest rev outside the
+ * transaction and another commit moved it before this one ran, so nothing was
+ * written. Patch ids are deterministic, so sending the same patches again is
+ * safe, and every round of a race has a winner: two `ix map` runs against one
+ * backend (two repos, two editors, a benchmark at concurrency 2) drain each
+ * other's contention within a few backed-off attempts. Without this, the loser
+ * counted the whole chunk as failed and left it for the next run -- 477 to
+ * 1,730 files of a 1,200-2,500-file repo missing from each of two concurrent
+ * maps, which then exited 1.
+ *
+ * Returns the last result; a caller that still sees `BaseRevMismatch` has
+ * exhausted `maxRetries` and falls back to counting the chunk as failed.
+ */
+export async function retryOnBaseRevRace<T extends { status?: string }>(
+ fn: () => Promise<T>,
+ maxRetries: number,
+ onRetry?: (attempt: number) => void,
+): Promise<T> {
+ for (let attempt = 0; ; attempt++) {
+ const result = await fn();
+ if (result.status !== COMMIT_STATUS_BASE_REV_MISMATCH || attempt >= maxRetries) return result;
+ onRetry?.(attempt + 1);
+ // Jittered so two racing runs stop re-colliding in lockstep. The window is
+ // the other side's transaction, which for a bulk runs to seconds.
+ const delay = 50 * (1 << Math.min(attempt, 5)) + Math.random() * 100;
+ await new Promise<void>(r => setTimeout(r, delay));
+ }
+}
+
// Every entry here means the same thing operationally: this batch was refused for
// being too big to apply as one unit, so committing fewer patches per request can
// still succeed. That is what makes bisecting the right recovery.
@@ -2150,6 +2182,13 @@ export async function ingestFiles(
const COMMIT_HTTP_MAX_FILES = parsePositiveIntEnv('IX_COMMIT_HTTP_MAX_FILES', 1000); // files per HTTP request to the backend
const COMMIT_CONCURRENCY = parsePositiveIntEnv('IX_COMMIT_CONCURRENCY', 8); // parallel HTTP save requests
const COMMIT_CONFLICT_RETRIES = parsePositiveIntEnv('IX_COMMIT_CONFLICT_RETRIES', 6); // retry transient Arango lock conflicts
+ // Re-sends of a commit that lost the base-rev race to another writer; see retryOnBaseRevRace.
+ const COMMIT_BASE_REV_RETRIES = parsePositiveIntEnv('IX_COMMIT_BASE_REV_RETRIES', 8);
+ let baseRevRetries = 0;
+ const noteBaseRevRetry = (): void => {
+ baseRevRetries++;
+ if (debug) process.stderr.write(`\n [base-rev race] another writer moved the rev; re-sending\n`);
+ };
const YIELD_EVERY = 100; // yield event loop every N files during parse
if (debug) {
@@ -2349,7 +2388,11 @@ export async function ingestFiles(
`commit ${item.fileNumber} of ${totalFiles} ${nodePath.basename(item.filePath)} (per-file)`
);
const commitStart = performance.now();
- const result = await retryOnConflict(() => client.commitPatch(item.patch), COMMIT_CONFLICT_RETRIES);
+ const result = await retryOnBaseRevRace(
+ () => retryOnConflict(() => client.commitPatch(item.patch), COMMIT_CONFLICT_RETRIES),
+ COMMIT_BASE_REV_RETRIES,
+ noteBaseRevRetry,
+ );
const chunkMs = Math.round(performance.now() - commitStart);
recordMs(chunkMs);
timings.fallbackCommitMs += chunkMs;
@@ -2500,9 +2543,13 @@ export async function ingestFiles(
const rest = probeQueue.slice(chunk.length);
const bulkStart = performance.now();
try {
- const result = await retryOnConflict(
- () => client.commitPatchBulk(chunk.map(item => item.patch)),
- COMMIT_CONFLICT_RETRIES,
+ const result = await retryOnBaseRevRace(
+ () => retryOnConflict(
+ () => client.commitPatchBulk(chunk.map(item => item.patch)),
+ COMMIT_CONFLICT_RETRIES,
+ ),
+ COMMIT_BASE_REV_RETRIES,
+ noteBaseRevRetry,
);
const ms = Math.round(performance.now() - bulkStart);
recordDrainMs(ms);
@@ -2523,7 +2570,7 @@ export async function ingestFiles(
commitErrors += chunk.length;
if (debug) {
process.stderr.write(
- `\n [cutoff] bulk lost the base-rev race; ${chunk.length} patches wrote nothing and will be re-sent next run\n`
+ `\n [cutoff] bulk lost the base-rev race ${COMMIT_BASE_REV_RETRIES + 1} times; ${chunk.length} patches wrote nothing and will be re-sent next run\n`
);
}
probeQueue = rest;
@@ -2687,9 +2734,13 @@ export async function ingestFiles(
`commit ${first.fileNumber}-${last.fileNumber} of ${totalFiles} ending ${nodePath.basename(last.filePath)}`
);
const commitStart = performance.now();
- const result = await retryOnConflict(
- () => client.commitPatchBulk(items.map(item => item.patch)),
- COMMIT_CONFLICT_RETRIES,
+ const result = await retryOnBaseRevRace(
+ () => retryOnConflict(
+ () => client.commitPatchBulk(items.map(item => item.patch)),
+ COMMIT_CONFLICT_RETRIES,
+ ),
+ COMMIT_BASE_REV_RETRIES,
+ noteBaseRevRetry,
);
const chunkMs = Math.round(performance.now() - commitStart);
commitMsPerChunk[ci] += chunkMs;
@@ -2708,7 +2759,7 @@ export async function ingestFiles(
commitErrors += items.length;
if (debug) {
process.stderr.write(
- `\n [commit lost the base-rev race] ${items.length} patches wrote nothing; they will be re-sent next run\n`
+ `\n [commit lost the base-rev race ${COMMIT_BASE_REV_RETRIES + 1} times] ${items.length} patches wrote nothing; they will be re-sent next run\n`
);
}
return;
@@ -3372,6 +3423,9 @@ export async function ingestFiles(
}
const committed = performance.now();
+ if (debug && baseRevRetries > 0) {
+ process.stderr.write(`\n [base-rev race] ${baseRevRetries} commit(s) re-sent after another writer moved the rev\n`);
+ }
// Persist mtime cache so next run can skip unchanged files quickly.
// Only save when no parse errors (avoid poisoning cache on partial failures).
@@ -3947,7 +4001,7 @@ async function ingestGitHub(opts: {
intent: `GitHub ingestion: ${repo.owner}/${repo.repo}`,
};
- const result = await client.commitPatch(patch);
+ const result = await retryOnBaseRevRace(() => client.commitPatch(patch), 8);
const elapsed = ((performance.now() - start) / 1000).toFixed(2);
if (opts.format === 'json') {
diff --git a/skills/ix/references/troubleshooting.md b/skills/ix/references/troubleshooting.md
index cba30428..d188e3fc 100644
--- a/skills/ix/references/troubleshooting.md
+++ b/skills/ix/references/troubleshooting.md
@@ -49,6 +49,10 @@ again.
- `IX_COMMIT_FAILURE_LIMIT=N` — consecutive failed commits before `ix map`
stops sending more (default 5). `0` sends every patch regardless, which is
the pre-#560 behaviour.
+- `IX_COMMIT_BASE_REV_RETRIES=N` — how many times a commit is re-sent after
+ another writer (a second `ix map` on the same backend) moved the graph
+ revision under it (default 8). Only a commit that loses every retry is
+ counted as failed and left for the next run.
## Harness presence (hermetic reproduction)