Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(ingest): stop turning one failed bulk commit into one doomed request per patch (#572)

Joseph Mikhail2fc0e009975f
diff --git a/ix-cli/src/cli/__tests__/commit-breaker.test.ts b/ix-cli/src/cli/__tests__/commit-breaker.test.ts
new file mode 100644
index 00000000..9a153818
--- /dev/null
+++ b/ix-cli/src/cli/__tests__/commit-breaker.test.ts
@@ -0,0 +1,614 @@
+import { describe, expect, it } from "vitest";
+import {
+ commitFailureLimit,
+ createCommitBreaker,
+ createDrainGate,
+ describeCommitCutoff,
+ drainFailureBudget,
+ drainInPasses,
+ perFileAction,
+} from "../commit-breaker.js";
+
+describe("commitFailureLimit", () => {
+ it("defaults to 5", () => {
+ expect(commitFailureLimit(undefined)).toBe(5);
+ expect(commitFailureLimit("")).toBe(5);
+ });
+
+ it("takes a caller's limit", () => {
+ expect(commitFailureLimit("2")).toBe(2);
+ });
+
+ it("treats 0 as a deliberate opt-out, not as a bad value", () => {
+ // The difference matters: falling back to the default here would make
+ // IX_COMMIT_FAILURE_LIMIT=0 silently keep the cutoff, so a user who needs
+ // every patch attempted has no way to say so.
+ expect(commitFailureLimit("0")).toBe(0);
+ });
+
+ it("reads an absurdly large value as 'never trip', not as the default", () => {
+ // It matches /^\d+$/ and the intent is unmistakable. Falling back gave them
+ // the TIGHTEST cutoff instead -- the one direction where guessing wrong
+ // loses writes.
+ expect(commitFailureLimit("99999999999999999999")).toBe(Number.MAX_SAFE_INTEGER);
+ });
+
+ it("falls back on a value that is not a non-negative integer", () => {
+ expect(commitFailureLimit("-1")).toBe(5);
+ expect(commitFailureLimit("many")).toBe(5);
+ // parseInt stopped at the first non-digit, so "0.5" read as 0 -- silently
+ // DISABLING the cutoff, which is the opposite of a conservative fallback.
+ expect(commitFailureLimit("0.5")).toBe(5);
+ });
+
+ it("falls back on the shapes parseInt and Number each got wrong", () => {
+ // parseInt read "1e3" as 1 -- a cutoff tripping on the very first failure --
+ // and Number reads it as 1000. Neither is what someone typing it meant, and
+ // guessing is worse than the documented default. Same for the values Number
+ // quietly coerces to 0, which would DISABLE the cutoff: the opposite of a
+ // conservative fallback.
+ expect(commitFailureLimit("1e3")).toBe(5);
+ expect(commitFailureLimit("0x10")).toBe(5);
+ expect(commitFailureLimit(" ")).toBe(5);
+ expect(commitFailureLimit(" 3 ")).toBe(5);
+ });
+});
+
+describe("createCommitBreaker", () => {
+ it("does not trip before the limit is reached", () => {
+ const b = createCommitBreaker(3);
+ b.recordFailure(new Error("a"));
+ b.recordFailure(new Error("b"));
+ expect(b.tripped()).toBe(false);
+ expect(b.consecutiveFailures()).toBe(2);
+ });
+
+ it("trips on the Nth consecutive failure", () => {
+ const b = createCommitBreaker(3);
+ for (const m of ["a", "b", "c"]) b.recordFailure(new Error(m));
+ expect(b.tripped()).toBe(true);
+ });
+
+ it("counts CONSECUTIVE failures, so one success clears the streak", () => {
+ // The distinction is the whole point. A handful of patches that fail for
+ // their own reasons, interleaved with commits that land, is a backend that
+ // is working — abandoning the rest of the run there would lose real work.
+ const b = createCommitBreaker(3);
+ b.recordFailure(new Error("a"));
+ b.recordFailure(new Error("b"));
+ b.recordSuccess();
+ b.recordFailure(new Error("c"));
+ b.recordFailure(new Error("d"));
+ expect(b.tripped()).toBe(false);
+ expect(b.consecutiveFailures()).toBe(2);
+ });
+
+ it("never trips at limit 0", () => {
+ const b = createCommitBreaker(0);
+ for (let i = 0; i < 100; i++) b.recordFailure(new Error(String(i)));
+ expect(b.tripped()).toBe(false);
+ });
+
+ it("stays tripped across further failures", () => {
+ const b = createCommitBreaker(2);
+ b.recordFailure(new Error("a"));
+ b.recordFailure(new Error("b"));
+ expect(b.tripped()).toBe(true);
+ b.recordFailure(new Error("c"));
+ expect(b.tripped()).toBe(true);
+ });
+
+ it("un-trips on a success, because that is direct evidence the backend is alive", () => {
+ // A permanent latch is the worse bug now that patches are held and re-sent
+ // rather than abandoned: after one trip, a backend that recovered completely
+ // still had every DELETION patch dropped for the rest of the run, since
+ // those never go through the bulk path that would have proved it healthy.
+ const b = createCommitBreaker(2);
+ b.recordFailure(new Error("a"));
+ b.recordFailure(new Error("b"));
+ expect(b.tripped()).toBe(true);
+
+ b.recordSuccess();
+ expect(b.tripped()).toBe(false);
+ expect(b.consecutiveFailures()).toBe(0);
+ });
+
+ it("counts the patches abandoned because it was already tripped", () => {
+ const b = createCommitBreaker(1);
+ b.recordFailure(new Error("a"));
+ b.recordSkipped(17);
+ b.recordSkipped();
+ expect(b.skipped()).toBe(18);
+ });
+});
+
+describe("everTripped", () => {
+ it("is false until the limit is reached", () => {
+ const b = createCommitBreaker(3);
+ b.recordFailure(new Error("x"));
+ b.recordFailure(new Error("x"));
+ expect(b.everTripped()).toBe(false);
+ });
+
+ it("stays true after a success un-trips the breaker", () => {
+ // This is the whole difference from `tripped()`. The diagnosis has to print
+ // for a run that gave up and then recovered: the 12-file reproduction at
+ // the default limit held 7 patches, the drain placed all 7, and gating on
+ // the leftovers meant the user saw the generic "all 12 failed to commit,
+ // re-run with --verbose" and no explanation of why `ix doctor` passes.
+ const b = createCommitBreaker(2);
+ b.recordFailure(new Error("x"));
+ b.recordFailure(new Error("x"));
+ expect(b.tripped()).toBe(true);
+
+ b.recordSuccess();
+ expect(b.tripped()).toBe(false);
+ expect(b.everTripped()).toBe(true);
+ });
+
+ it("survives reset, which only clears the current streak", () => {
+ const b = createCommitBreaker(1);
+ b.recordFailure(new Error("x"));
+ b.reset();
+ expect(b.tripped()).toBe(false);
+ expect(b.everTripped()).toBe(true);
+ });
+
+ it("never becomes true at limit 0", () => {
+ const b = createCommitBreaker(0);
+ for (let i = 0; i < 50; i++) b.recordFailure(new Error("x"));
+ expect(b.everTripped()).toBe(false);
+ });
+});
+
+describe("reset, which is what the end-of-run retry runs on", () => {
+ it("un-latches and clears the streak", () => {
+ const b = createCommitBreaker(2);
+ b.recordFailure(new Error("a"));
+ b.recordFailure(new Error("b"));
+ expect(b.tripped()).toBe(true);
+
+ b.reset();
+ expect(b.tripped()).toBe(false);
+ expect(b.consecutiveFailures()).toBe(0);
+ });
+
+ it("keeps skipped() cumulative, because the report spans the whole run", () => {
+ const b = createCommitBreaker(1);
+ b.recordFailure(new Error("a"));
+ b.recordSkipped(7);
+ b.reset();
+ b.recordSkipped(3);
+ expect(b.skipped()).toBe(10);
+ });
+
+ it("re-trips after the same number of failures, so the retry stays bounded", () => {
+ // This is why the retry is not a bypass: a backend that really is refusing
+ // everything gets N more attempts, not one per held-back patch.
+ const b = createCommitBreaker(3);
+ for (const m of ["a", "b", "c"]) b.recordFailure(new Error(m));
+ b.reset();
+
+ b.recordFailure(new Error("d"));
+ b.recordFailure(new Error("e"));
+ expect(b.tripped()).toBe(false);
+ b.recordFailure(new Error("f"));
+ expect(b.tripped()).toBe(true);
+ });
+
+ it("lets a success during the retry keep it un-tripped", () => {
+ // The false-trip case: the held-back patches are fine, so they commit and
+ // the streak never rebuilds.
+ const b = createCommitBreaker(2);
+ b.recordFailure(new Error("a"));
+ b.recordFailure(new Error("b"));
+ b.reset();
+
+ b.recordFailure(new Error("c"));
+ b.recordSuccess();
+ b.recordFailure(new Error("d"));
+ expect(b.tripped()).toBe(false);
+ });
+});
+
+describe("perFileAction", () => {
+ // Every revision of this decision that could not be driven directly grew a
+ // defect only an end-to-end run would show. These are those defects.
+
+ it("sends when nothing says otherwise", () => {
+ expect(perFileAction({ replay: false, tripped: false })).toBe("send");
+ expect(perFileAction({ replay: false, tripped: false, budgetLeft: 3 })).toBe("send");
+ });
+
+ it("holds on a trip when the caller has no budget", () => {
+ expect(perFileAction({ replay: false, tripped: true })).toBe("hold");
+ });
+
+ it("ignores the streak when the caller gave a budget", () => {
+ // The bug this exists for: with both live, the streak trips at `limit` long
+ // before a budget of `2 * limit` can bind, so the budget was inert and the
+ // drain still stranded every good patch sitting behind a run of bad ones.
+ expect(perFileAction({ replay: false, tripped: true, budgetLeft: 3 })).toBe("send");
+ });
+
+ it("holds once the budget is spent", () => {
+ expect(perFileAction({ replay: false, tripped: false, budgetLeft: 0 })).toBe("hold");
+ expect(perFileAction({ replay: false, tripped: true, budgetLeft: 0 })).toBe("hold");
+ });
+
+ it("stops a REPLAY on a spent budget by counting it, never by holding it", () => {
+ // Stopping is right -- a 409 naming 999 landed ids would otherwise send 999
+ // serialized doomed commits while holding the fallback mutex -- but HOLDING
+ // is not how to stop. The only consumer turns everything it holds into a
+ // commit error, so this used to report writes the server had CONFIRMED as
+ // failures and suppress the mtime baseline over them.
+ expect(perFileAction({ replay: true, tripped: false, budgetLeft: 0 })).toBe("count-applied");
+ expect(perFileAction({ replay: true, tripped: true, budgetLeft: 0 })).toBe("count-applied");
+ });
+
+ it("stops a replay on a trip by counting it, never by holding it", () => {
+ // Those patches are confirmed landed by the server's own 409 body. Holding
+ // them would drop them from the counters and let the cutoff report "The
+ // graph is unchanged" for a run whose patches are all in the graph.
+ expect(perFileAction({ replay: true, tripped: true })).toBe("count-applied");
+ });
+
+ it("still sends a replay while nothing has given up", () => {
+ // An earlier revision folded the replay check into the tripped check and
+ // inverted it, so nothing was ever held back at all.
+ expect(perFileAction({ replay: true, tripped: false })).toBe("send");
+ });
+});
+
+describe("drainInPasses", () => {
+ /**
+ * A driver standing in for the per-file loop.
+ *
+ * The budget defaults to `drainFailureBudget()` -- the number `ingest.ts`
+ * actually passes -- and every test below uses it. An earlier version of this
+ * file hardcoded 10 while the CLI ran at 5, so the stranding guarantee was
+ * asserted only in a regime that does not exist and a review found it failing
+ * in the one that does.
+ */
+ function driver(bad: ReadonlySet<string>, budget = drainFailureBudget()) {
+ const sent: string[] = [];
+ const placed: string[] = [];
+ const attempt = async (items: string[]) => {
+ let left = budget;
+ let placedHere = 0;
+ for (let i = 0; i < items.length; i++) {
+ sent.push(items[i]);
+ if (bad.has(items[i])) {
+ left--;
+ if (left <= 0) return { placed: placedHere > 0, unreached: items.slice(i + 1) };
+ } else {
+ placed.push(items[i]);
+ placedHere++;
+ }
+ }
+ return { placed: placedHere > 0, unreached: [] };
+ };
+ return { attempt, sent, placed };
+ }
+
+ const range = (prefix: string, n: number): string[] =>
+ Array.from({ length: n }, (_, i) => `${prefix}${i + 1}`);
+
+ it("uses the budget the CLI passes", () => {
+ // The whole point of sharing this: a test that picks its own budget can
+ // pass while the shipped configuration loses writes.
+ expect(drainFailureBudget(commitFailureLimit())).toBe(5);
+ expect(drainFailureBudget(0)).toBe(1);
+ });
+
+ it("walks the whole set from the far end when nothing fails", async () => {
+ const d = driver(new Set());
+ expect(await drainInPasses(["a", "b", "c"], d.attempt)).toEqual([]);
+ // Reversed: the fan-out stopped INSIDE whatever run of bad patches tripped
+ // it, so the rest of that run is the first thing in the held set.
+ expect(d.sent).toEqual(["c", "b", "a"]);
+ });
+
+ it("stops a dead backend after three passes — tail, head, middle", async () => {
+ // The amplification #560 is about. Three samples, none of them placing
+ // anything, is the bar; two only ever cover the ends.
+ const held = range("p", 500);
+ const d = driver(new Set(held));
+ const leftover = await drainInPasses(held, d.attempt);
+
+ expect(d.sent).toHaveLength(3 * drainFailureBudget());
+ expect(leftover).toHaveLength(500 - 3 * drainFailureBudget());
+ });
+
+ it("strands nothing when clusters sit at BOTH ends of the held set", async () => {
+ // Two passes can only sample the two ends, so `[b x 5, g x 400, B x 10]`
+ // defeated the previous rule outright: the tail pass walks into B, the head
+ // pass into b, neither places anything, and all 400 committable patches
+ // between them were handed back unsent -- on a healthy backend, and then
+ // repeated on every later run, since the mtime baseline is not written when
+ // a run has commit errors. The third pass starts in the middle.
+ const bad = new Set([...range("b", 5), ...range("B", 10)]);
+ const held = [...range("b", 5), ...range("g", 400), ...range("B", 10)];
+ const d = driver(bad);
+
+ expect(await drainInPasses(held, d.attempt)).toEqual([]);
+ expect(d.placed).toHaveLength(400);
+ });
+
+ it("does not treat a pass that placed nothing as proof, when it stopped EARLY", async () => {
+ // Held [g1..g400, b1..b15] reverses to [b15..b1, g400..g1], so the first
+ // pass meets the bad patches immediately and places nothing. Stopping there
+ // reported all 400 committable patches as errors without sending one, and
+ // since the mtime baseline is not written on a run with commit errors the
+ // next `ix map` reproduced it exactly, forever.
+ const held = [...range("g", 400), ...range("b", 15)];
+ const d = driver(new Set(range("b", 15)));
+
+ expect(await drainInPasses(held, d.attempt)).toEqual([]);
+ expect(d.placed).toHaveLength(400);
+ });
+
+ it("strands nothing when the bad patches form TWO clusters", async () => {
+ const bad = [...range("b", 15)];
+ const held = [...bad.slice(0, 5), ...range("g", 400), ...bad.slice(5), ...range("h", 485)];
+ const d = driver(new Set(bad));
+
+ expect(await drainInPasses(held, d.attempt)).toEqual([]);
+ expect(d.placed).toHaveLength(885);
+ });
+
+ it("strands nothing for a cluster LONGER than the budget", async () => {
+ const held = [...range("g", 400), ...range("B", 25), ...range("h", 485)];
+ const d = driver(new Set(range("B", 25)));
+
+ expect(await drainInPasses(held, d.attempt)).toEqual([]);
+ expect(d.placed).toHaveLength(885);
+ });
+
+ it("strands nothing with FAR more bad patches than any pass cap would allow", async () => {
+ // The case a pass cap could not survive, and the reason there is no longer
+ // one. Capped at three passes of five, the walk could get past at most 15
+ // failing patches; a 1,000-patch held set with 20 scattered bad ones placed
+ // 686 and handed back 294 committable patches unsent -- a net loss against
+ // `main`, which fans out and commits all 980.
+ const bad = new Set(range("b", 20));
+ const held = Array.from({ length: 1000 }, (_, i) =>
+ i % 50 === 49 ? `b${Math.floor(i / 50) + 1}` : `g${i}`,
+ );
+ const d = driver(bad);
+
+ const leftover = await drainInPasses(held, d.attempt);
+
+ expect(leftover).toEqual([]);
+ expect(d.placed.filter(x => !bad.has(x))).toHaveLength(980);
+ // And it costs no more requests than `main` would have sent.
+ expect(d.sent.length).toBeLessThanOrEqual(held.length);
+ });
+
+ it("strands nothing when clusters alternate with the budget the whole way", async () => {
+ const bad = new Set(range("b", 24));
+ const held = [
+ ...Array.from({ length: 4 }, (_, k) => [...range("g", 50).map(x => `${x}_${k}`), ...range("b", 6).map((_, i) => `b${k * 6 + i + 1}`)]).flat(),
+ ...range("z", 50),
+ ];
+ const d = driver(bad);
+
+ expect(await drainInPasses(held, d.attempt)).toEqual([]);
+ expect(d.placed.filter(x => !bad.has(x))).toHaveLength(250);
+ });
+
+ it("keeps going while the backend takes anything, at no more cost than main", async () => {
+ // The trade-off, pinned so it is a decision rather than a surprise. A
+ // backend that accepts one write and refuses the other 994 is never stopped
+ // by the drain: each pass eats `budget` of the refusing patches, so it runs
+ // ~199 passes. That is not a regression -- `main` sends one request per
+ // patch for all 995 and has no cutoff at all -- and the alternative, a cap
+ // on failures, hands back committable patches unsent, which is a real loss
+ // where this is only slowness.
+ const held = range("p", 995);
+ const bad = new Set(held.slice(1));
+ const d = driver(bad);
+
+ await drainInPasses(held, d.attempt);
+
+ expect(d.placed).toEqual([held[0]]);
+ // Never more than one request per held patch.
+ expect(d.sent.length).toBeLessThanOrEqual(held.length);
+ // And no patch is attempted twice.
+ expect(new Set(d.sent).size).toBe(d.sent.length);
+ });
+
+ it("does not call the backend at all for an empty held set", async () => {
+ const d = driver(new Set());
+ expect(await drainInPasses([], d.attempt)).toEqual([]);
+ expect(d.sent).toEqual([]);
+ });
+});
+
+describe("createDrainGate", () => {
+ it("opens for the first held set of the run", () => {
+ expect(createDrainGate().shouldDrain(0)).toBe(true);
+ });
+
+ it("stays open after one drain that placed nothing", () => {
+ // One miss is a few-second fast-rejecting blip. Closing on it strands the
+ // next batch's held patches with no drain at all.
+ const gate = createDrainGate();
+ gate.record(false, 0);
+ expect(gate.shouldDrain(0)).toBe(true);
+ });
+
+ it("closes after two in a row that placed nothing", () => {
+ const gate = createDrainGate();
+ gate.record(false, 0);
+ gate.record(false, 0);
+ expect(gate.shouldDrain(0)).toBe(false);
+ });
+
+ it("a drain that placed something clears the streak", () => {
+ const gate = createDrainGate();
+ gate.record(false, 0);
+ gate.record(true, 12);
+ gate.record(false, 12);
+ expect(gate.shouldDrain(12)).toBe(true);
+ });
+
+ it("REOPENS once a patch lands after it closed", () => {
+ // The finding this exists for. A thirty-second blip in batches 3 and 4
+ // closed the gate for the whole run, so a hold at batch 30 -- against a
+ // backend that had since committed nineteen thousand patches -- had its
+ // patches counted as errors and never sent. The reset used to live inside
+ // the branch the flag disables, so nothing could ever reopen it.
+ const gate = createDrainGate();
+ gate.record(false, 100);
+ gate.record(false, 100);
+ expect(gate.shouldDrain(100)).toBe(false);
+
+ expect(gate.shouldDrain(19_000)).toBe(true);
+ });
+
+ it("lets one drain through periodically, because a closed gate blocks its own reopen", () => {
+ // The reopen condition is a patch landing, and with the drain off the only
+ // remaining source of that is a BULK success. A backend that recovers while
+ // its bulks keep failing for GROUP reasons -- an old memory layer answering
+ // "partially committed" with no parseable ids, or a tail of oversized files
+ // bisecting to single-item 413s -- could then never clear the gate, and
+ // every remaining patch in the run was reported as an error with nothing
+ // sent, where `main` would have committed them one at a time.
+ const gate = createDrainGate(2, 3);
+ gate.record(false, 0);
+ gate.record(false, 0);
+
+ expect(gate.shouldDrain(0)).toBe(false);
+ expect(gate.shouldDrain(0)).toBe(false);
+ expect(gate.shouldDrain(0), "the third batch gets a probe").toBe(true);
+ // ...and then it is closed again for another interval.
+ expect(gate.shouldDrain(0)).toBe(false);
+ });
+
+ it("re-closes if the backend goes back to placing nothing", () => {
+ const gate = createDrainGate();
+ gate.record(false, 100);
+ gate.record(false, 100);
+ expect(gate.shouldDrain(19_000)).toBe(true);
+
+ gate.record(false, 19_000);
+ gate.record(false, 19_000);
+ expect(gate.shouldDrain(19_000)).toBe(false);
+ });
+});
+
+describe("describeCommitCutoff", () => {
+ it("names the count, the endpoint, what was not sent, and the error", () => {
+ const b = createCommitBreaker(2);
+ b.recordFailure(new Error("500: transaction begin timeout"));
+ b.recordFailure(new Error("500: transaction begin timeout"));
+ b.recordSkipped(17);
+
+ const msg = describeCommitCutoff(b, "http://localhost:8090");
+ // The configured limit, not the live streak: requests already in flight when
+ // the breaker tripped keep landing and keep incrementing it, so the streak
+ // is whatever the race produced and does not match IX_COMMIT_FAILURE_LIMIT.
+ // Deliberately no count in the headline. The fan-out stops on a streak of
+ // `limit`, the drain on a per-pass budget of `limit`, and this banner
+ // prints for both -- so any single number it names is wrong for one of them.
+ b.recordFailure(new Error("a straggler that landed after the decision"));
+ expect(msg).not.toContain("consecutive");
+ expect(msg).toContain("kept failing");
+ expect(msg).toContain("http://localhost:8090");
+ expect(msg).toContain("17 patches were not sent");
+ expect(msg).toContain("transaction begin timeout");
+ expect(msg).toContain("IX_COMMIT_FAILURE_LIMIT=0");
+ });
+
+ it("does not claim the graph is unchanged when patches landed before it stopped", () => {
+ // A backend can start refusing part-way through a large ingest. Saying "the
+ // graph is unchanged" then contradicts the summary printed right beneath.
+ const b = createCommitBreaker(1);
+ b.recordFailure(new Error("x"));
+ b.recordSkipped(3);
+
+ expect(describeCommitCutoff(b, "e", 0)).toContain("The graph is unchanged.");
+
+ const partial = describeCommitCutoff(b, "e", 40);
+ expect(partial).not.toContain("The graph is unchanged");
+ expect(partial).toContain("40 other patches did land");
+ // No ordering claim. A cutoff holds patches and the run continues, so there
+ // is no single moment it "stopped": a run that tripped in batch 1,
+ // recovered, and committed 19,000 more would have claimed all 19,000
+ // landed "before it stopped".
+ expect(partial).not.toContain("before it stopped");
+ });
+
+ it("names both causes rather than sending everyone to the database", () => {
+ // The memory layer answers 500 for a saturated ArangoDB AND for a patch
+ // body it will not accept -- verified against the released image -- so the
+ // CLI cannot tell them apart. Asserting the database is the cause sends a
+ // user with a rejected patch to `docker stats` and `/_api/query/current`.
+ const b = createCommitBreaker(1);
+ b.recordFailure(new Error("500: {\"error\":\"internal_error\"}"));
+ b.recordSkipped(2);
+
+ const msg = describeCommitCutoff(b, "http://localhost:8090", 5);
+ expect(msg).toContain("Either the database cannot keep up");
+ expect(msg).toContain("rejecting these");
+ expect(msg).toContain("says which");
+ });
+
+ it("drops the unsent line entirely when the drain placed everything", () => {
+ // The banner prints whenever the cutoff FIRED, and the drain may since have
+ // sent every patch it held. Saying "0 patches were not sent", or claiming
+ // the run stopped committing, would both be false on the one message whose
+ // job is saying what happened.
+ const b = createCommitBreaker(1);
+ b.recordFailure(new Error("500: transaction begin timeout"));
+
+ const msg = describeCommitCutoff(b, "http://localhost:8090", 12);
+ expect(b.skipped()).toBe(0);
+ expect(msg).not.toContain("not sent");
+ expect(msg).not.toContain("Stopped committing");
+ expect(msg).toContain("kept failing");
+ expect(msg).toContain("transaction begin timeout");
+ });
+
+ it("drops the unsent attribution when the run also ran out of time", () => {
+ // The deadline branch of describeCommitOutcome owns what to do about the
+ // missing patches, and the two must not contradict. A cutoff that fired
+ // BEFORE the clock has already called recordSkipped, so without this the
+ // banner printed "N were not sent - sending them one at a time would have
+ // added load to a backend that is already the reason they fail" directly
+ // above "Ingest ran out of time: raise IX_MAP_DEADLINE_MS". The diagnosis
+ // is still worth printing; the attribution of the count is not.
+ const b = createCommitBreaker(1);
+ b.recordFailure(new Error("500: transaction begin timeout"));
+ b.recordSkipped(400);
+
+ const msg = describeCommitCutoff(b, "e", 0, true);
+ expect(msg).not.toContain("added load");
+ expect(msg).toContain("transaction begin timeout");
+ // But it still NAMES the count. Dropping the line entirely handed the
+ // cutoff's 400 to the clock, and the deadline message that prints next
+ // points at IX_MAP_DEADLINE_MS -- the wrong fix for patches deliberately
+ // withheld from a refusing backend.
+ expect(msg).toContain("400 of them were withheld by this cutoff");
+ expect(describeCommitCutoff(b, "e", 0, false)).toContain("400 patches were not sent");
+ });
+
+ it("says 'patch was' for one and 'patches were' for many", () => {
+ const one = createCommitBreaker(1);
+ one.recordFailure(new Error("x"));
+ one.recordSkipped(1);
+ expect(describeCommitCutoff(one, "e")).toContain("1 patch was not sent");
+ });
+
+ it("truncates a long body rather than pasting an HTML error page into the summary", () => {
+ // Ix#528: the observed failure was a proxy 504 whose body is a full HTML
+ // page. describeStitchFailure already refuses to echo those; this message
+ // quotes the error because the exact text is the diagnostic, so it caps it.
+ const b = createCommitBreaker(1);
+ b.recordFailure(new Error("x".repeat(5000)));
+ const msg = describeCommitCutoff(b, "e");
+ expect(msg.length).toBeLessThan(1000);
+ expect(msg).toContain("…");
+ });
+});
diff --git a/ix-cli/src/cli/__tests__/ingest-commit-outcome.test.ts b/ix-cli/src/cli/__tests__/ingest-commit-outcome.test.ts
index 717b34e6..9ab2f4c7 100644
--- a/ix-cli/src/cli/__tests__/ingest-commit-outcome.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-commit-outcome.test.ts
@@ -107,6 +107,118 @@ describe("ingestCompletedCleanly", () => {
});
});
+describe("describeCommitOutcome when the deadline fired", () => {
+ it("does not tell the user to raise the budget when NOTHING committed", () => {
+ // With a 5-minute per-request timeout and a 15-minute budget, a stalled
+ // backend fits ~3 attempts in -- the cutoff's streak cannot reach its limit,
+ // so the run ends on the deadline. "Raise IX_MAP_DEADLINE_MS" is then advice
+ // to wait longer on a backend that answered nothing.
+ const out = describeCommitOutcome(20, 0, "--verbose", true, 0);
+ expect(out.kind).toBe("fatal");
+ if (out.kind === "ok") throw new Error("expected a message");
+ expect(out.message).toContain("without committing anything");
+ // Both readings are live: the deadline covers discovery, hashing and parse
+ // too, so a large repo can spend it before the first commit is attempted --
+ // and there raising it IS the fix. Name both rather than assert one.
+ expect(out.message).toContain("raise IX_MAP_DEADLINE_MS");
+ expect(out.message).toContain("not accepting writes");
+ expect(out.message).toContain("ix doctor");
+ });
+
+ it("still points at the budget when work was getting done", () => {
+ const out = describeCommitOutcome(20, 4000, "--verbose", true, 0);
+ expect(out.kind).toBe("warn");
+ if (out.kind === "ok") throw new Error("expected a message");
+ expect(out.message).toContain("Raise IX_MAP_DEADLINE_MS");
+ });
+
+ it("reports the deadline ahead of the cutoff when both happened", () => {
+ // A run that skipped a handful to the cutoff and then lost thousands to the
+ // clock lost far more to the clock; blaming the cutoff hides the number that
+ // would change the outcome.
+ const out = describeCommitOutcome(5000, 100, "--verbose", true, 7);
+ if (out.kind === "ok") throw new Error("expected a message");
+ expect(out.message).toContain("Raise IX_MAP_DEADLINE_MS");
+ expect(out.message).not.toContain("never sent");
+ });
+});
+
+describe("describeCommitOutcome when both the cutoff and the deadline fired", () => {
+ it("names the cutoff's share instead of blaming the clock for all of it", () => {
+ // 200 patches withheld from a refusing backend, then 3 more lost to the
+ // clock. Reporting all 203 as "abandoned when the map deadline fired ...
+ // raise IX_MAP_DEADLINE_MS" is the wrong remedy for 200 of them, and it
+ // contradicts the banner above, which names the split.
+ const out = describeCommitOutcome(203, 50, "--verbose", true, 200);
+ if (out.kind === "ok") throw new Error("expected a message");
+
+ expect(out.message).toContain("3 of 253 file patches were abandoned when the map deadline fired");
+ expect(out.message).toContain("200 more were withheld earlier by the commit cutoff");
+ expect(out.message).toContain("see the cutoff above");
+ });
+
+ it("keeps the plain wording when the cutoff withheld nothing", () => {
+ const out = describeCommitOutcome(3, 50, "--verbose", true, 0);
+ if (out.kind === "ok") throw new Error("expected a message");
+ expect(out.message).toContain("3 of 53 file patches were abandoned when the map deadline fired");
+ expect(out.message).not.toContain("withheld earlier");
+ });
+});
+
+describe("describeCommitOutcome with a commit cutoff", () => {
+ it("points at the cutoff even when the drain placed every held patch", () => {
+ // The cutoff fired, so the banner above has already named the endpoint and
+ // quoted the backend's error -- but nothing was left unsent, and gating on
+ // that count sent the reader to the generic "Re-run with --debug to see
+ // why" directly beneath a message that had just told them why. The 12-file
+ // reproduction at the default limit lands here: five fan-out failures trip
+ // it, the drain sends the remaining seven, none are "never sent".
+ const out = describeCommitOutcome(12, 0, "--debug", false, 0, true);
+ expect(out.kind).toBe("fatal");
+ if (out.kind === "ok") throw new Error("expected a message");
+ expect(out.message).toContain("See the cutoff above");
+ expect(out.message).not.toContain("never sent");
+ // And it must still point AT the flag here. Every one of these errors came
+ // from a patch that was sent and did emit a per-file line, so the sibling
+ // message's "the flag will not add to it" -- true when patches were held
+ // back unsent -- would steer the reader away from the only detail there is.
+ expect(out.message).toContain("--debug lists the individual failures");
+ expect(out.message).not.toContain("will not add to it");
+ });
+
+ it("still names the unsent count when there is one", () => {
+ const out = describeCommitOutcome(12, 0, "--debug", false, 3, true);
+ if (out.kind === "ok") throw new Error("expected a message");
+ expect(out.message).toContain("3 of them never sent");
+ });
+
+ it("points at the cutoff instead of a flag that would show nothing", () => {
+ // The cutoff has already printed the endpoint, the streak and the error. A
+ // patch that was never sent produces no per-file line, so the usual advice
+ // sends the reader to --verbose for output that does not exist.
+ const out = describeCommitOutcome(20, 0, "--verbose", false, 15);
+ expect(out.kind).toBe("fatal");
+ if (out.kind === "ok") throw new Error("expected a message");
+ expect(out.message).toContain("15 of them never sent");
+ expect(out.message).toContain("See the cutoff above");
+ expect(out.message).not.toContain("to see why");
+ });
+
+ it("warns rather than fails when some patches landed first", () => {
+ const out = describeCommitOutcome(15, 40, "--verbose", false, 15);
+ expect(out.kind).toBe("warn");
+ });
+
+ it("leaves the ordinary messages alone when nothing was cut off", () => {
+ const none = describeCommitOutcome(3, 0, "--verbose", false, 0);
+ const some = describeCommitOutcome(3, 10, "--verbose", false, 0);
+ expect(none.kind).toBe("fatal");
+ expect(some.kind).toBe("warn");
+ if (none.kind !== "ok") expect(none.message).toContain("Ingest committed nothing");
+ if (some.kind !== "ok") expect(some.message).toContain("Re-run 'ix map' to retry");
+ });
+});
+
describe("describeStitchFailure", () => {
it("reports incomplete cross-repository relationships without dumping an HTML response", () => {
const message = describeStitchFailure(new Error("504: <html>\nlarge proxy response"));
diff --git a/ix-cli/src/cli/__tests__/ingest-retry.test.ts b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
index 69db9a53..db56d572 100644
--- a/ix-cli/src/cli/__tests__/ingest-retry.test.ts
+++ b/ix-cli/src/cli/__tests__/ingest-retry.test.ts
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from 'vitest';
import {
commitBulkWithPayloadSplit,
+ commitFailureIndictsBackend,
isAbortError,
isBulkPartiallyCommittedError,
isPayloadTooLargeError,
@@ -78,6 +79,64 @@ describe('isPayloadTooLargeError', () => {
});
});
+describe('commitFailureIndictsBackend', () => {
+ // Only failures that say something about the BACKEND may feed the run-wide
+ // cutoff, because the cutoff abandons every remaining patch in the run.
+
+ it('counts a per-request timeout — a backend that HANGS produces nothing else', () => {
+ // The bug this test exists for: `isAbortError` is true for TimeoutError,
+ // and IxClient builds every commit signal from AbortSignal.timeout(5min).
+ // Excluding aborts therefore made the cutoff inert against a stalled
+ // ArangoDB, which is the saturation shape it was written for. A backend is
+ // not obliged to answer 500; when it does not, the timeout IS the signal.
+ const timeout = Object.assign(new Error('The operation was aborted due to timeout'), {
+ name: 'TimeoutError',
+ });
+ expect(isAbortError(timeout)).toBe(true);
+ expect(commitFailureIndictsBackend(timeout, false)).toBe(true);
+ });
+
+ it('does not count the run deadline, which is our clock and not the backend', () => {
+ const aborted = Object.assign(new Error('aborted'), { name: 'AbortError' });
+ expect(commitFailureIndictsBackend(aborted, true)).toBe(false);
+ // Anything at all, once the budget is gone.
+ expect(commitFailureIndictsBackend(new Error('500: nope'), true)).toBe(false);
+ });
+
+ it('counts a lock conflict that has already exhausted its retries', () => {
+ // An earlier revision excluded these, to protect two overlapping `ix map`
+ // runs. It was an over-correction twice over: `timeout waiting to lock key`
+ // and `error: 1200` are what a RocksDB-backed ArangoDB emits under the very
+ // saturation this cutoff is for, and because those patterns are ALSO in the
+ // retry list, excluding them multiplied every doomed patch by retryOnConflict
+ // instead of stopping it. By the time an error reaches this function, six
+ // backed-off attempts have already been spent on it.
+ for (const msg of ['write-write conflict', 'timeout waiting to lock key', 'Error: 1200 bad']) {
+ expect(isRetryableCommitConflict(new Error(msg)), msg).toBe(true);
+ expect(commitFailureIndictsBackend(new Error(msg), false), msg).toBe(true);
+ }
+ });
+
+ it('counts a transport failure too', () => {
+ for (const msg of ['fetch failed', 'read ECONNRESET', 'connect ECONNREFUSED 127.0.0.1:8090']) {
+ expect(commitFailureIndictsBackend(new Error(msg), false), msg).toBe(true);
+ }
+ });
+
+ it('does not count payload-too-large, which is about the patch and not the server', () => {
+ // Five oversized generated files in a row must not stop the whole repo.
+ expect(commitFailureIndictsBackend(new Error('413: payload too large'), false)).toBe(false);
+ expect(commitFailureIndictsBackend(ARANGO_TRANSACTION_LIMIT_ERROR, false)).toBe(false);
+ });
+
+ it('counts an ordinary backend rejection, including one that says "aborted"', () => {
+ // `isAbortError` matches "aborted" anywhere in the text, so this 500 used to
+ // be excluded from the cutoff along with the real aborts.
+ expect(commitFailureIndictsBackend(new Error('500: internal server error'), false)).toBe(true);
+ expect(commitFailureIndictsBackend(new Error('500: {"error":"AQL: transaction aborted"}'), false)).toBe(true);
+ });
+});
+
describe('commitBulkWithPayloadSplit', () => {
it('bisects rejected payloads until every smaller bulk commit succeeds', async () => {
const bulkCalls: number[][] = [];
@@ -160,6 +219,126 @@ describe('commitBulkWithPayloadSplit', () => {
expect(commitIndividually).toHaveBeenCalledOnce();
expect(commitIndividually).toHaveBeenCalledWith([1, 2, 3], error);
});
+
+ // Ix#560. The per-file fallback is right for a bulk that failed for a reason
+ // specific to the GROUP; it is exactly wrong when the backend is refusing
+ // every write, because it becomes one doomed request per patch, serialized,
+ // against the backend that is the reason. `beforeFallback` lets the caller
+ // say which case this is.
+ it('abandons the per-file fallback when the caller has given up on the backend', async () => {
+ const error = new Error('500: transaction begin timeout');
+ const commitIndividually = vi.fn(async () => {});
+ const onAbandoned = vi.fn();
+ let stop = false;
+
+ await commitBulkWithPayloadSplit([1, 2, 3], {
+ commitBulk: async () => { stop = true; throw error; },
+ onBulkCommitted: vi.fn(),
+ commitIndividually,
+ shouldStop: () => stop,
+ onAbandoned,
+ });
+
+ expect(commitIndividually).not.toHaveBeenCalled();
+ expect(onAbandoned).toHaveBeenCalledWith([1, 2, 3], error);
+ });
+
+ it('still fans out while the caller has not given up', async () => {
+ const error = new Error('500: internal server error');
+ const commitIndividually = vi.fn(async () => {});
+
+ await commitBulkWithPayloadSplit([1, 2, 3], {
+ commitBulk: async () => { throw error; },
+ onBulkCommitted: vi.fn(),
+ commitIndividually,
+ shouldStop: () => false,
+ });
+
+ expect(commitIndividually).toHaveBeenCalledWith([1, 2, 3], error);
+ });
+
+ it('still sends the BULK once the caller has given up, and only skips the fan-out', async () => {
+ // One request per group is not the amplification -- the fan-out is, at one
+ // per patch -- and it is the only way a backend that recovered mid-run can
+ // prove it. Skipping it was also strictly worse: the group came back as N
+ // serialized per-file commits via the retry instead of one bulk.
+ const commitBulk = vi.fn(async () => { throw new Error('500: still refusing'); });
+ const commitIndividually = vi.fn(async () => {});
+ const onAbandoned = vi.fn();
+
+ await commitBulkWithPayloadSplit([1, 2, 3], {
+ commitBulk,
+ onBulkCommitted: vi.fn(),
+ commitIndividually,
+ shouldStop: () => true,
+ onAbandoned,
+ });
+
+ expect(commitBulk).toHaveBeenCalledOnce();
+ expect(commitIndividually).not.toHaveBeenCalled();
+ expect(onAbandoned).toHaveBeenCalled();
+ });
+
+ it('lets a recovered backend commit the group even after the caller gave up', async () => {
+ // The reason the bulk is still sent: `shouldStop` says the RUN gave up, and
+ // this is how that decision gets revisited.
+ const commitBulk = vi.fn(async () => 'ok');
+ const onBulkCommitted = vi.fn();
+
+ await commitBulkWithPayloadSplit([1, 2, 3], {
+ commitBulk,
+ onBulkCommitted,
+ commitIndividually: vi.fn(async () => {}),
+ shouldStop: () => true,
+ });
+
+ expect(commitBulk).toHaveBeenCalledOnce();
+ expect(onBulkCommitted).toHaveBeenCalledOnce();
+ });
+
+ it('sends both halves of a payload split even after the caller gives up', async () => {
+ // Two bulk requests, not two fan-outs. Skipping the second half sent it to
+ // the retry as N serialized per-file commits, which is the amplification
+ // this cutoff exists to stop -- and denied a recovered backend the chance
+ // to commit it in one request.
+ let stop = false;
+ const sent: number[][] = [];
+
+ await commitBulkWithPayloadSplit([1, 2, 3, 4], {
+ commitBulk: async (batch) => {
+ if (batch.length > 2) throw new Error('413: payload too large');
+ sent.push(batch);
+ stop = true; // the first half is what gives up
+ throw new Error('500: transaction begin timeout');
+ },
+ onBulkCommitted: vi.fn(),
+ commitIndividually: vi.fn(async () => {}),
+ shouldStop: () => stop,
+ onAbandoned: vi.fn(),
+ });
+
+ expect(sent).toEqual([[1, 2], [3, 4]]);
+ });
+
+ it('bisects a payload-too-large group without consulting shouldStop again mid-split', async () => {
+ // A bisect is a DIFFERENT, smaller request that can succeed. It is not a
+ // backend failure, and nothing about it should stop the run: a >1,000-file
+ // repo relies on it (Ix#516).
+ const committed: number[][] = [];
+
+ await commitBulkWithPayloadSplit([1, 2, 3, 4], {
+ commitBulk: async (batch) => {
+ if (batch.length > 2) throw new Error('413: payload too large');
+ committed.push(batch);
+ return 'ok';
+ },
+ onBulkCommitted: vi.fn(),
+ commitIndividually: vi.fn(async () => {}),
+ shouldStop: () => false,
+ });
+
+ expect(committed).toEqual([[1, 2], [3, 4]]);
+ });
});
// The body the server actually sends, as the client stringifies it:
@@ -202,6 +381,26 @@ describe('partly-committed bulk groups', () => {
expect(parseBulkCommittedPatchIds(new Error('409: {"committed_patch_ids":["p1",7]}'))).toBeUndefined();
});
+ it('marks the landed replay as a replay, so a caller that has given up still sends it', async () => {
+ // Those patches are CONFIRMED landed. Skipping them would count writes the
+ // server already has as commit errors and drop them from patchesApplied.
+ const commitIndividually = vi.fn(async () => {});
+ const items = [item('a'), item('b')];
+
+ await commitBulkWithPayloadSplit(items, {
+ commitBulk: async (batch) => {
+ if (batch.length === 2) throw partialBody(['a'], 2);
+ return 'ok';
+ },
+ onBulkCommitted: vi.fn(),
+ commitIndividually,
+ patchIdOf: (i) => i.patch.patchId,
+ shouldStop: () => false,
+ });
+
+ expect(commitIndividually).toHaveBeenCalledWith([items[0]], undefined, { replay: true });
+ });
+
it('replays what landed and re-bulks only what is missing', async () => {
const items = ['p1', 'p2', 'p3', 'p4', 'p5'].map(item);
const bulkCalls: string[][] = [];
@@ -268,7 +467,9 @@ describe('partly-committed bulk groups', () => {
});
expect(commitBulk).toHaveBeenCalledOnce();
- expect(commitIndividually).toHaveBeenCalledWith(items, error);
+ // Flagged as a replay: the server has CONFIRMED it holds all of them, so a
+ // tripped commit cutoff must not count them as failures (Ix#560).
+ expect(commitIndividually).toHaveBeenCalledWith(items, error, { replay: true });
});
it('keeps the old whole-batch fallback when the caller cannot identify patches', async () => {
diff --git a/ix-cli/src/cli/commands/ingest.ts b/ix-cli/src/cli/commands/ingest.ts
index b979d359..910eecd3 100644
--- a/ix-cli/src/cli/commands/ingest.ts
+++ b/ix-cli/src/cli/commands/ingest.ts
@@ -18,6 +18,14 @@ import { loadIngestionModules } from './ingestion-loader.js';
import { ensureWorkspaceIdState } from '../bootstrap.js';
import { detectSystem, repoWorkspaceIdFor, lookupPackage, readPackageNames, readPackageDeps } from '../system.js';
import { CLIENT_EXPECTED_SCHEMA_VERSION } from '../backend-status.js';
+import {
+ createCommitBreaker,
+ createDrainGate,
+ describeCommitCutoff,
+ drainFailureBudget,
+ drainInPasses,
+ perFileAction,
+} from '../commit-breaker.js';
import { readBackendHealth } from './upgrade.js';
import { SUPPORTED_EXTENSIONS } from '../supported-extensions.js';
import { canRenderProgress } from '../stderr.js';
@@ -528,7 +536,13 @@ export function planDeletedFileRecovery(
return { previousDeletedFiles, nextDeletedFiles, recreatedPaths, forceReingestPaths };
}
+/** Lock contention, as opposed to the transport failures below. Kept separate
+ * only so each group's reason for being retryable stays readable. */
const COMMIT_CONFLICT_RETRY_PATTERNS = [
+ // ArangoDB lock conflicts. These were briefly split into their own constant
+ // so `commitFailureIndictsBackend` could exclude them; that exclusion was
+ // reverted in round 3 -- it made the cutoff inert in a real saturation mode --
+ // and the split had no consumer left, so it is inlined.
'write-write conflict',
'timeout waiting to lock key',
'error: 1200',
@@ -603,6 +617,42 @@ export function isRetryableCommitConflict(err: unknown): boolean {
return COMMIT_CONFLICT_RETRY_PATTERNS.some(pattern => message.includes(pattern));
}
+/**
+ * Does this commit failure say anything about the BACKEND? (Ix#560)
+ *
+ * Only failures that indict the backend may feed the run-wide cutoff, because
+ * the cutoff abandons every remaining patch in the run.
+ *
+ * Excluded, and each for its own reason:
+ *
+ * - the run's own deadline. That is our wall clock, not the backend's
+ * answer, and `describeCommitOutcome` has a whole branch to keep them
+ * apart. Note this is the DEADLINE, not `isAbortError`: a per-request
+ * `AbortSignal.timeout` also raises TimeoutError, and a backend that
+ * HANGS produces nothing else. Excluding those made the cutoff inert in
+ * exactly the saturation shape it exists for -- a stalled ArangoDB is not
+ * obliged to answer 500, and when it does not, the timeout IS the signal.
+ * - payload-too-large, which is about this patch and not the server; five
+ * oversized generated files in a row must not stop the whole repo.
+ *
+ * A lock conflict is NOT excluded, though an earlier revision excluded it.
+ * `timeout waiting to lock key` and `error: 1200` are what a RocksDB-backed
+ * ArangoDB emits under exactly the sustained write contention this cutoff is
+ * for, so excluding them made it inert in a real saturation mode -- and, worse,
+ * those same patterns are in the RETRY list, so every doomed patch was being
+ * multiplied by `retryOnConflict` instead of stopped.
+ *
+ * The worry that motivated the exclusion -- two overlapping `ix map` runs
+ * contending -- is already answered upstream of here: by the time an error
+ * reaches this function, `retryOnConflict` has spent six backed-off attempts
+ * on it. A conflict that survives all six is not transient contention.
+ */
+export function commitFailureIndictsBackend(err: unknown, runDeadlineExpired: boolean): boolean {
+ if (runDeadlineExpired) return false;
+ if (isPayloadTooLargeError(err)) return false;
+ return true;
+}
+
async function retryOnConflict<T>(fn: () => Promise<T>, maxRetries: number): Promise<T> {
for (let attempt = 0; ; attempt++) {
try {
@@ -650,15 +700,53 @@ export async function commitBulkWithPayloadSplit<T, R>(
handlers: {
commitBulk: (batch: T[]) => Promise<R>;
onBulkCommitted: (batch: T[], result: R) => void;
- commitIndividually: (batch: T[], error: unknown) => Promise<void>;
+ commitIndividually: (batch: T[], error: unknown, opts?: { replay?: boolean }) => Promise<void>;
onSplit?: (batch: T[], error: unknown) => void;
/** Patch id of an item, so a partly-committed group can be resumed. */
patchIdOf?: (item: T) => string | undefined;
onPartialBulk?: (landed: T[], missing: T[], error: unknown) => void;
+ /**
+ * Has the caller given up on the backend? (Ix#560)
+ *
+ * Consulted before the per-file fallback, and ONLY there. The bulk is
+ * always sent: one request per group is not the amplification, and it is
+ * how a backend that recovered mid-run gets to prove it.
+ * The fallback exists for a bulk that failed for a
+ * reason specific to the GROUP (too large, partly applied), where sending
+ * the patches separately is a different request that can succeed. When the
+ * backend itself is refusing writes, none of that holds: the fan-out is one
+ * doomed request per patch, serialized, against the very backend that is
+ * the reason.
+ *
+ * Absent, nothing is ever abandoned: the behaviour that shipped.
+ */
+ shouldStop?: () => boolean;
+ /** `batch` was abandoned unsent. `error` is absent when none was seen. */
+ onAbandoned?: (batch: T[], error?: unknown) => void;
},
): Promise<void> {
if (items.length === 0) return;
+ // No cutoff check before a bulk that is actually a GROUP. One request per
+ // group is not the amplification -- the per-file fallback is, at one request
+ // per patch -- and sending it is the only way a backend that has recovered
+ // mid-run can prove it and let the rest of the ingest through. Skipping it
+ // was strictly worse than useless: the group went to the deferred set and
+ // came back as N SERIALIZED per-file commits instead of one bulk.
+ //
+ // A SINGLE-patch chunk is different, and the difference is the whole issue.
+ // At `IX_COMMIT_HTTP_MAX_FILES=1` -- which #560's reporters tried, and which
+ // the header's own table measures -- every chunk is one patch, so "one
+ // request per group" IS one request per patch and the rule above halves
+ // main's cost instead of bounding it: a 10,000-file map against a dead
+ // backend still sent ~10,000 doomed bulks. The cutoff applies to those, and
+ // recovery is still provable -- the drain's gate lets one probe through every
+ // few batches for exactly this.
+ if (items.length === 1 && handlers.shouldStop?.()) {
+ handlers.onAbandoned?.(items);
+ return;
+ }
+
let result: R;
try {
result = await handlers.commitBulk(items);
@@ -683,16 +771,34 @@ export async function commitBulkWithPayloadSplit<T, R>(
const idOf = handlers.patchIdOf;
const landed = items.filter(item => { const id = idOf(item); return id !== undefined && landedIds.has(id); });
const missing = items.filter(item => { const id = idOf(item); return id === undefined || !landedIds.has(id); });
- // Only worth it when the split is real. If nothing (or everything) is
- // claimed as landed, re-bulking `missing` would repeat this exact call.
+ // The server says it already holds every one of them. Re-bulking would
+ // repeat this exact call, so this still falls through to the per-file
+ // path -- but flagged as a replay, because that is what it is. Without the
+ // flag, a tripped breaker counts patches the server has CONFIRMED as
+ // commit errors, the same over-reporting `replay` exists to prevent on
+ // the sibling branch (Ix#560).
+ if (landed.length === items.length && items.length > 0) {
+ await handlers.commitIndividually(items, err, { replay: true });
+ return;
+ }
+ // Only worth it when the split is real. If nothing is claimed as landed,
+ // re-bulking `missing` would repeat this exact call.
if (landed.length > 0 && missing.length > 0) {
handlers.onPartialBulk?.(landed, missing, err);
- await handlers.commitIndividually(landed, undefined);
+ // `replay: true` -- the server has confirmed it already holds these.
+ // Re-sending them is bookkeeping, not a write the breaker should
+ // refuse: skipping them would count patches that DID land as commit
+ // errors and drop them from patchesApplied (Ix#495).
+ await handlers.commitIndividually(landed, undefined, { replay: true });
await commitBulkWithPayloadSplit(missing, handlers);
return;
}
}
+ if (handlers.shouldStop?.()) {
+ handlers.onAbandoned?.(items, err);
+ return;
+ }
await handlers.commitIndividually(items, err);
return;
}
@@ -872,7 +978,15 @@ export function describeCommitOutcome(
*/
debugFlag: string = "--debug",
/** True when the run was cut short by the map deadline rather than rejected. */
- deadlineHit: boolean = false
+ deadlineHit: boolean = false,
+ /** Patches abandoned unsent by the commit cutoff (Ix#560). */
+ cutoffSkipped: number = 0,
+ /**
+ * Did the cutoff fire at any point, whether or not it ended up losing
+ * patches? When it did, the banner above has already given the reason, and
+ * this line must not send the reader back to a flag that adds nothing.
+ */
+ cutoffFired: boolean = cutoffSkipped > 0
): CommitOutcome {
if (commitErrors <= 0) return { kind: "ok" };
const attempted = commitErrors + patchesApplied;
@@ -882,11 +996,92 @@ export function describeCommitOutcome(
// patch is a backend problem, a deadline is our own wall clock and the fix
// is a longer budget or a smaller repo. Folding both into "failed to
// commit" is the same information loss this change exists to undo.
+ //
+ // Unless nothing landed at all. With a 5-minute per-request timeout and a
+ // 15-minute budget a stalled backend only fits ~3 attempts in, so the
+ // cutoff's streak cannot reach its limit and the run ends on the deadline
+ // -- and telling someone to RAISE the budget then is advice to wait longer
+ // on a backend that answered nothing. A run that committed zero patches
+ // did not run out of time doing work; it ran out of time waiting.
+ if (patchesApplied === 0) {
+ // Deliberately NOT "raising the budget will not help": the deadline
+ // covers discovery, hashing and parsing too, so a large repo or a short
+ // IX_MAP_DEADLINE_MS can spend it before the first commit is even
+ // attempted -- and there raising it is exactly the fix. Both readings
+ // are live from here, so name both rather than assert the wrong one.
+ // The cutoff's share is named here too. This is the LIKELIEST shape for a
+ // dead backend, not a corner: nothing lands, so telling the user all N
+ // were abandoned by the clock -- and to lengthen the budget -- is the
+ // wrong remedy for the ones deliberately never sent, and contradicts the
+ // banner directly above, which has just named the split.
+ const withheld =
+ cutoffSkipped > 0
+ ? `${commitErrors - cutoffSkipped} file patches were abandoned when the map deadline fired and ` +
+ `${cutoffSkipped} more were withheld by the commit cutoff, and none landed first`
+ : `all ${commitErrors} file patches were abandoned when the map deadline fired, and none landed first`;
+ return {
+ kind: "fatal",
+ message:
+ `Ingest ran out of time without committing anything: ${withheld}. Either the budget ran out ` +
+ `before the commits did (raise IX_MAP_DEADLINE_MS or map a smaller path), or the backend ` +
+ `is not accepting writes — an ArangoDB that cannot begin a transaction is still reachable ` +
+ `and still consistent, so \`ix doctor\` passes. \`docker stats\` and ` +
+ `\`/_api/query/current\` tell the two apart.`,
+ };
+ }
+ // The cutoff's share is named rather than folded in. A run that withheld
+ // 200 patches from a refusing backend and then lost 3 more to the clock
+ // reported all 203 as "abandoned when the map deadline fired ... raise
+ // IX_MAP_DEADLINE_MS", which is the wrong remedy for 200 of them. The
+ // banner above already names the split; this line was the one contradicting
+ // it.
+ const clockLost = commitErrors - cutoffSkipped;
+ const share =
+ cutoffSkipped > 0
+ ? `${clockLost} of ${attempted} file patches were abandoned when the map deadline fired, and ` +
+ `${cutoffSkipped} more were withheld earlier by the commit cutoff`
+ : `${commitErrors} of ${attempted} file patches were abandoned when the map deadline fired`;
+ const remedy =
+ cutoffSkipped > 0
+ ? `The graph is missing all of them. Raise IX_MAP_DEADLINE_MS or map a smaller path for the first group; ` +
+ `see the cutoff above for the second.`
+ : `The graph is missing those files. Raise IX_MAP_DEADLINE_MS or map a smaller path.`;
+ return {
+ kind: "warn",
+ message: `Ingest ran out of time: ${share}. ${remedy}`,
+ };
+ }
+
+ // After the deadline branch, deliberately: a run that tripped the cutoff on
+ // a handful of patches and THEN blew its budget on thousands lost far more
+ // to the clock, and attributing all of it to the cutoff hides the number
+ // that would actually change the outcome.
+ if (cutoffFired) {
+ // Two shapes, and they want opposite advice about the debug flag.
+ //
+ // When patches were held back unsent, they produce no per-file line at all,
+ // so pointing at the flag sends the reader to output that does not exist --
+ // the cutoff banner above is the only account of them there is.
+ //
+ // When the drain placed everything it held, `cutoffSkipped` is 0 and every
+ // counted error came from a patch that WAS sent and DID emit a line. The
+ // banner names only the last error and no filenames, so telling the reader
+ // the flag adds nothing steers them away from the only per-file detail
+ // available. That is the shape the 12-file reproduction produces at the
+ // default limit: five sent failures, seven drained successfully.
+ if (cutoffSkipped === 0) {
+ return {
+ kind: patchesApplied === 0 ? "fatal" : "warn",
+ message:
+ `Ingest stopped early: ${commitErrors} of ${attempted} file patches are missing from the graph. ` +
+ `See the cutoff above for why it stopped; ${debugFlag} lists the individual failures.`,
+ };
+ }
return {
kind: patchesApplied === 0 ? "fatal" : "warn",
message:
- `Ingest ran out of time: ${commitErrors} of ${attempted} file patches were abandoned when ` +
- `the map deadline fired. The graph is missing those files. Raise IX_MAP_DEADLINE_MS or map a smaller path.`,
+ `Ingest stopped early: ${commitErrors} of ${attempted} file patches are missing from the graph, ` +
+ `${cutoffSkipped} of them never sent. See the cutoff above for why; ${debugFlag} will not add to it.`,
};
}
@@ -1230,6 +1425,36 @@ export async function ingestFiles(
let commitErrors = 0;
let stitchErrors = 0;
let stitchError: unknown;
+ // Ix#560: a backend refusing every commit gets one failed request per
+ // patch out of the per-file fallback. Stop after a streak instead.
+ const commitBreaker = createCommitBreaker();
+ /**
+ * Whether a batch's held-back patches get a drain. Run-scoped: the decision
+ * spans batches, and it has to be revocable -- see `createDrainGate`.
+ */
+ const drainGate = createDrainGate();
+ /**
+ * Patches the BACKEND accepted, counted only where a request came back OK.
+ *
+ * Not `patchesApplied`, which the gate used to read. That counter also rises
+ * on the `count-applied` path, where up to a thousand patches are counted
+ * from a 409 body without a single request being sent -- so a full-landed
+ * replay could reopen a gate that had closed against a dead backend and
+ * restore the per-batch re-probing the gate exists to prevent.
+ */
+ let patchesTheBackendTook = 0;
+ /**
+ * Commit failures that said something about the BACKEND rather than about one
+ * patch. Counted so the drain gate can tell "refused everything because the
+ * backend is down" from "refused everything because every patch was too big".
+ */
+ let backendIndictingFailures = 0;
+ /**
+ * The RUN deadline, captured here because `commitPreparedPatches` shadows
+ * `opts` with its own. Not `isAbortError`: a per-request AbortSignal.timeout
+ * raises TimeoutError too, and that is a saturation signal, not our clock.
+ */
+ const runDeadlineExpired = (): boolean => opts.deadlineSignal?.aborted === true;
let tooLarge = 0;
let minifiedLikely = 0;
let outsideRoot = 0;
@@ -1480,6 +1705,20 @@ export async function ingestFiles(
): Promise<number> => {
if (preparedPatches.length === 0) return 0;
+ /**
+ * Patches the cutoff held back rather than sent (Ix#560).
+ *
+ * Held, not discarded. Abandoning them outright is what made an earlier
+ * version able to wedge a workspace permanently: the mtime baseline is
+ * not written on a run with commit errors, so the next `ix map`
+ * re-ingests in the same order, trips at the same point, and abandons the
+ * same patches forever. They get one ordinary pass at the end IF the
+ * backend has meanwhile proved it accepts writes -- see the drain below.
+ */
+ const deferredByCutoff: PreparedPatch[] = [];
+ /** Held by the run deadline rather than the cutoff -- see the branch below. */
+ const deferredByDeadline: PreparedPatch[] = [];
+
const chunks: PreparedPatch[][] = [];
let bulkChunk: PreparedPatch[] = [];
const flushBulkChunk = (): void => {
@@ -1499,18 +1738,46 @@ export async function ingestFiles(
flushBulkChunk();
const commitMsPerChunk = new Array<number>(chunks.length).fill(0);
+ // The drain belongs to no chunk, but its time is still commit time --
+ // a no-op sink here dropped it from the reported total.
+ let drainMs = 0;
+ const recordDrainMs = (ms: number): void => { drainMs += ms; };
let nextChunk = 0;
// Fallback mutex: serializes per-file commits across all workers so that
// concurrent fallback loops don't race on revisions.current (Error 1200).
let fallbackTail: Promise<void> = Promise.resolve();
- const runChunk = async (ci: number): Promise<void> => {
- const chunk = chunks[ci];
- const endFile = chunk[chunk.length - 1].fileNumber;
- const commitIndividually = async (
- items: PreparedPatch[],
- bulkError?: unknown,
- ): Promise<void> => {
+ /**
+ * Commit `items` one at a time, serialized against every other per-file
+ * loop in this run.
+ *
+ * Hoisted out of runChunk so the end-of-run drain runs the SAME loop
+ * rather than a second copy of it -- the counters, the mutex and the
+ * breaker interaction are exactly the parts that must not drift.
+ * `recordMs` is the only thing that was chunk-scoped.
+ */
+ const commitItemsSerially = async (
+ items: PreparedPatch[],
+ recordMs: (ms: number) => void,
+ bulkError?: unknown,
+ fallbackOpts?: {
+ replay?: boolean;
+ /**
+ * Total failures this loop may absorb before giving up, replacing the
+ * breaker's CONSECUTIVE streak for callers that need to get past a
+ * run of bad patches rather than stop at one.
+ *
+ * A streak cannot: N adjacent patches the backend rejects on their
+ * own merits look exactly like a dead backend for as long as the run
+ * of them lasts, so any streak of N or fewer stops inside the
+ * cluster and strands everything after it -- and the mtime baseline,
+ * never written on a run with commit errors, makes the next `ix map`
+ * repeat it exactly. A budget keeps going past the cluster and only
+ * stops when the FAILURES add up, which is the thing worth bounding.
+ */
+ failureBudget?: number;
+ },
+ ): Promise<void> => {
if (debug && bulkError !== undefined) {
const first = items[0];
const last = items[items.length - 1];
@@ -1525,8 +1792,74 @@ export async function ingestFiles(
let resolveThis!: () => void;
fallbackTail = new Promise<void>(r => { resolveThis = r; });
await prev;
+ let budgetLeft = fallbackOpts?.failureBudget ?? Number.POSITIVE_INFINITY;
+ /** Failed re-sends of server-confirmed patches. Bounds the replay, locally. */
+ let replayFailures = 0;
try {
- for (const item of items) {
+ for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+ const item = items[itemIndex];
+ const isReplay = fallbackOpts?.replay === true;
+ if (runDeadlineExpired()) {
+ // A replay first: these patches are confirmed landed by the
+ // server's own 409 body, so the clock stops the BOOKKEEPING and
+ // not a write. Holding them would push up to a thousand
+ // confirmed-committed patches into the commit-error count,
+ // report the graph as missing files it holds, and suppress the
+ // mtime baseline over them -- the exact over-reporting the
+ // `replay` flag exists to prevent, reintroduced by putting this
+ // deadline check above it.
+ if (isReplay) {
+ patchesApplied += items.length - itemIndex;
+ break;
+ }
+ // Every request from here is rejected by `fetch` on an already
+ // aborted signal, so the fan-out is pure waste -- and it is not
+ // otherwise stopped, because `commitFailureIndictsBackend`
+ // returns false for everything once the deadline has fired.
+ // That is right for CLASSIFYING (the deadline is our clock, not
+ // the backend's answer) and it left the cutoff and the drain
+ // budget both inert for the tail of the run, so a bulk failure
+ // at that point fanned out to one serialized commit per patch
+ // with nothing able to stop it.
+ //
+ // A SEPARATE list from `deferredByCutoff`, deliberately. These
+ // patches were abandoned by the clock, not by the backend, and
+ // feeding them to `recordSkipped` made the cutoff banner blame
+ // the backend for them -- printing "400 patches were not sent,
+ // sending them would have added load to a backend that is
+ // already the reason they fail" directly above "Ingest ran out
+ // of time: raise IX_MAP_DEADLINE_MS". Two contradictory
+ // diagnoses, and the wrong count on the first.
+ // `describeCommitOutcome` already orders the deadline ahead of
+ // the cutoff for exactly this reason.
+ for (let k = itemIndex; k < items.length; k++) deferredByDeadline.push(items[k]);
+ break;
+ }
+ const action = perFileAction({
+ replay: isReplay,
+ tripped: commitBreaker.tripped(),
+ budgetLeft: fallbackOpts?.failureBudget === undefined ? undefined : budgetLeft,
+ });
+ if (action === "hold") {
+ for (let k = itemIndex; k < items.length; k++) deferredByCutoff.push(items[k]);
+ break;
+ }
+ if (action === "count-applied") {
+ // Confirmed landed by the server's 409, and we have given up
+ // sending. They are still IN THE GRAPH, so dropping them from
+ // the counters would make the cutoff report "The graph is
+ // unchanged" for a run that changed it.
+ //
+ // This branch adds no commitErrors, and `persistIngestBaseline-
+ // IfClean` is gated on the RUN-WIDE count, so on a run whose
+ // only anomaly is this the baseline IS written and the next run
+ // skips these files. That is correct here and only here --
+ // the server's own 409 confirmed they landed -- but it is the
+ // opposite of what an earlier comment claimed, so it is worth
+ // stating rather than leaving to be re-derived.
+ patchesApplied += items.length - itemIndex;
+ break;
+ }
try {
// Move the label per file. It used to be set once, before the
// bulk attempt, and never again -- so a fallback of several
@@ -1538,10 +1871,12 @@ export async function ingestFiles(
const commitStart = performance.now();
const result = await retryOnConflict(() => client.commitPatch(item.patch), COMMIT_CONFLICT_RETRIES);
const chunkMs = Math.round(performance.now() - commitStart);
- commitMsPerChunk[ci] += chunkMs;
+ recordMs(chunkMs);
timings.fallbackCommitMs += chunkMs;
latestRev = advanceRev(latestRev, result.rev);
patchesApplied++;
+ patchesTheBackendTook++;
+ commitBreaker.recordSuccess();
opts?.onCommitted?.(item, result.rev);
} catch (commitErr) {
// Counted apart from parseErrors: a patch that parsed fine and
@@ -1549,7 +1884,51 @@ export async function ingestFiles(
// tree, which is a different problem from a file we could not
// read. Reporting it as a parse error hid that distinction in
// both the summary and the JSON contract.
- commitErrors++;
+ // Only failures that indict the BACKEND spend the budget --
+ // the same filter the breaker uses. A payload-too-large is
+ // about that one patch, and letting a handful of oversized
+ // generated files exhaust the drain would strand hundreds of
+ // perfectly committable patches on a healthy backend.
+ const indicts = commitFailureIndictsBackend(commitErr, runDeadlineExpired());
+ if (indicts) budgetLeft--;
+ // ...but a REPLAY never indicts the backend, whatever the error
+ // looks like: the server's own 409 confirmed those patches
+ // landed, so a failure re-sending one says nothing about its
+ // willingness to write. This counter is the drain gate's
+ // evidence, and its own doc says exactly that twenty lines
+ // below; counting replays here contradicted it.
+ if (indicts && !isReplay) backendIndictingFailures++;
+ if (isReplay) {
+ // Confirmed landed by the server's own 409 body. A failed
+ // re-send loses no write, so counting it as a commit error
+ // would let the cutoff claim "The graph is unchanged" for a
+ // run whose patches are all in the graph.
+ patchesApplied++;
+ // It still has to STOP, though. The full-landed branch of
+ // `commitBulkWithPayloadSplit` hands the whole chunk here and
+ // returns before its `shouldStop` check, so nothing else
+ // bounds it: an un-tripped breaker would watch up to
+ // IX_COMMIT_HTTP_MAX_FILES doomed commits go out one at a
+ // time behind the global fallback mutex, blocking every other
+ // chunk's fallback.
+ //
+ // Bounded LOCALLY, not through the run-wide breaker. Feeding
+ // it there was round 13's fix and it indicted the whole run
+ // for something the CLI knows says nothing about the backend's
+ // willingness to write: the server already confirmed these
+ // patches landed, so a backend that answers a hard error for
+ // re-committing one -- a legitimate idempotency
+ // implementation -- tripped the run-wide cutoff after five
+ // replays while perfectly healthy.
+ replayFailures++;
+ if (commitBreaker.limit > 0 && replayFailures >= commitBreaker.limit) {
+ patchesApplied += items.length - itemIndex - 1;
+ break;
+ }
+ } else {
+ commitErrors++;
+ if (indicts) commitBreaker.recordFailure(commitErr);
+ }
if (debug) {
const errMsg = String(commitErr);
const truncated = errMsg.length > 200 ? errMsg.slice(0, 200) + '…' : errMsg;
@@ -1560,7 +1939,225 @@ export async function ingestFiles(
} finally {
resolveThis();
}
- };
+ };
+
+ /**
+ * Re-send the patches the cutoff held back, in alternating passes.
+ *
+ * Not a bypass, but the bound is the BUDGET, not the breaker: with a
+ * failure budget in play `perFileAction` ignores the streak entirely, so
+ * saying "the breaker is active, so this stays bounded" -- as this
+ * comment used to -- described a mechanism that does not run. What each
+ * pass carries is `failureBudget` doomed requests. There is deliberately
+ * NO pass cap -- one would hand back committable patches unsent, a real
+ * loss where this is only slowness -- so the bound is the HELD-SET SIZE:
+ * a pass only attempts items no earlier pass reached, so the drain sends
+ * at most one request per held patch, which is what `main` sends anyway.
+ * A dead backend is stopped after `3 x budget` -- one per sample; one that
+ * accepts a little and refuses the rest is not stopped at all and pays the
+ * full size.
+ *
+ * The streak is still RESET per pass, for a different reason: five
+ * adjacent patches the backend rejects on their own merits are
+ * indistinguishable from a dead backend at the time, and the difference
+ * only shows when the next patch commits. Carrying the streak in would
+ * leave the run's banner reporting the hold that caused the drain rather
+ * than what the drain found.
+ *
+ * Without any of this the mtime baseline -- never written on a run with
+ * commit errors -- makes the next run repeat the same order, stop at the
+ * same point, and drop the same patches forever.
+ */
+ /** Re-send the held set. What it cannot place is left on `deferredByCutoff`. */
+ const commitIndividuallyAfterCutoff = async (itemsIn: PreparedPatch[]): Promise<void> => {
+ let items = itemsIn;
+ if (debug) {
+ process.stderr.write(
+ `\n [cutoff] re-sending ${items.length} held-back patches with a fresh streak\n`,
+ );
+ }
+
+ // One bulk first, because the held set is usually not the problem.
+ //
+ // `onAbandoned` hands over whole CHUNKS, not just the tail the fan-out
+ // reached, so on a backend that recovers by the end of the batch this
+ // would otherwise serialize thousands of single-patch commits where one
+ // request per chunk would do -- the Ix#495 shape, on the recovery path
+ // this design exists to serve. A backend that is still refusing costs
+ // one extra doomed request, the same price the per-chunk bulk already
+ // pays and deliberately never skips.
+ //
+ // DELETIONS are excluded. `runChunk` routes any patch with a
+ // DeleteNode/DeleteEdge op to its own single-item `client.commitPatch`,
+ // and `ingest-reconcile.test.ts` pins that -- the bulk endpoint is not
+ // trusted to apply delete ops. They fall through to the passes below,
+ // which commit them one at a time as their routing requires.
+ //
+ // CHUNKED at `COMMIT_HTTP_MAX_FILES`, not skipped above it. Skipping
+ // meant that lowering `IX_COMMIT_HTTP_MAX_FILES` -- which #560's
+ // reporters explicitly tried -- turned the probe off entirely and gave
+ // them the serialized fan-out it exists to avoid.
+ //
+ // A PARTIAL 409 is handled here rather than left to the passes, because
+ // the passes cannot: they run with a failure budget and never with
+ // `replay: true`, so re-sending a patch the server has already
+ // confirmed would be counted as a commit error -- suppressing the mtime
+ // baseline and reporting files as missing from a graph that holds them,
+ // which is the over-reporting `replay` exists to prevent (Ix#495).
+ const needsPerFile = (item: PreparedPatch): boolean => patchRequiresPerFileCommit(item.patch);
+ const stillToSend: PreparedPatch[] = items.filter(needsPerFile);
+ let probeQueue = items.filter(item => !needsPerFile(item));
+
+ while (probeQueue.length > 1) {
+ const chunk = probeQueue.slice(0, COMMIT_HTTP_MAX_FILES);
+ const rest = probeQueue.slice(chunk.length);
+ const bulkStart = performance.now();
+ try {
+ const result = await retryOnConflict(
+ () => client.commitPatchBulk(chunk.map(item => item.patch)),
+ COMMIT_CONFLICT_RETRIES,
+ );
+ const ms = Math.round(performance.now() - bulkStart);
+ recordDrainMs(ms);
+ timings.bulkCommitMs += ms;
+ latestRev = advanceRev(latestRev, result.rev);
+ patchesApplied += chunk.length;
+ patchesTheBackendTook += chunk.length;
+ commitBreaker.recordSuccess();
+ for (const item of chunk) opts?.onCommitted?.(item, result.rev);
+ if (debug) process.stderr.write(` [cutoff] bulk placed ${chunk.length} of ${items.length} held
+`);
+ probeQueue = rest;
+ } catch (bulkErr) {
+ // Record the time either way. The FAILING probe is the expensive
+ // one -- a saturated backend answers with a per-request timeout,
+ // multiplied by the conflict retries -- so dropping it hid minutes
+ // from the reported commit total on exactly the run this targets.
+ const ms = Math.round(performance.now() - bulkStart);
+ recordDrainMs(ms);
+ timings.bulkCommitMs += ms;
+ let unconfirmed = chunk;
+ let confirmedAny = false;
+ if (isBulkPartiallyCommittedError(bulkErr)) {
+ const landedIds = parseBulkCommittedPatchIds(bulkErr);
+ if (landedIds !== undefined) {
+ const landed = chunk.filter(item => landedIds.has(item.patch.patchId));
+ if (landed.length > 0) {
+ patchesApplied += landed.length;
+ // ...and through `onCommitted`, like every other "these
+ // landed" path. The deletion batch is the one caller that
+ // supplies it, and it uses it to write `durableDeletedFiles`:
+ // a deletion patch whose reconcile found nothing to remove
+ // carries no delete op, so it rides the bulk path, can be held
+ // by the cutoff, and can be 409-confirmed here -- counted
+ // applied but never recorded in the deleted-files baseline.
+ // `latestRev` rather than a rev of its own: the server holds
+ // these already, so their rev is at or below it, and
+ // `saveIngestBaseline` floors what it stores anyway.
+ for (const item of landed) opts?.onCommitted?.(item, latestRev);
+ // NOT `patchesTheBackendTook`. That counter's whole reason for
+ // existing is to keep 409-sourced counts from reopening a gate
+ // closed against a dead backend -- "counted only where a
+ // request came back OK", as its own doc says. A backend that
+ // can read but cannot begin a write transaction answers 409
+ // with ids it already holds, and on a re-run after a failed
+ // ingest the patch ids are identical, so feeding those here
+ // would reopen the gate and un-trip the breaker every batch.
+ confirmedAny = true;
+ if (debug) {
+ process.stderr.write(
+ ` [cutoff] bulk partly committed: ${landed.length} of ${chunk.length} already landed
+`,
+ );
+ }
+ }
+ unconfirmed = chunk.filter(item => !landedIds.has(item.patch.patchId));
+ }
+ }
+ // Keep the quoted error current, for the same reason `onAbandoned`
+ // does -- but NOT for a 409 that confirmed patches. That answer says
+ // the backend is taking writes, and the banner's `Last error:` is
+ // what the message and the troubleshooting row both tell the user
+ // distinguishes a saturated database from a rejected patch. A 409
+ // says neither, and quoting it would replace the TimeoutError that
+ // actually caused the cutoff. Not `recordFailure` either -- one bulk
+ // is one request, not N.
+ if (!confirmedAny && commitFailureIndictsBackend(bulkErr, runDeadlineExpired())) {
+ commitBreaker.noteError(bulkErr);
+ }
+ if (confirmedAny) {
+ // Positive proof the backend is taking writes, so KEEP PROBING --
+ // and re-bulk what it did NOT confirm rather than hand it to the
+ // passes, which have no bulk path. `commitBulkWithPayloadSplit`
+ // does exactly this with the same response: the unconfirmed set
+ // is a different group id, so one more request settles it where
+ // 497 serialized commits behind the global mutex would otherwise
+ // be the Ix#495 amplification this probe exists to prevent.
+ //
+ // Deliberately NOT `recordSuccess()`. A 409 naming ids the server
+ // already holds is not evidence it can take a NEW write -- that is
+ // the same reasoning that keeps those ids out of
+ // `patchesTheBackendTook` twenty lines above, and calling both
+ // would have made the two contradict each other. A backend that
+ // can read but cannot begin a write transaction answers exactly
+ // this way on a re-run, and un-tripping the run-wide cutoff there
+ // hands the next batch's failing bulk back to the fan-out.
+ probeQueue = [...unconfirmed, ...rest];
+ } else {
+ for (const item of unconfirmed) stillToSend.push(item);
+ // One real failure ends the probing: the rest goes to the passes,
+ // which is where a backend that is actually refusing belongs.
+ for (const item of rest) stillToSend.push(item);
+ probeQueue = [];
+ }
+ }
+ }
+ for (const item of probeQueue) stillToSend.push(item);
+ if (stillToSend.length === 0) return;
+ items = stillToSend;
+ // A budget, not the breaker's streak: the held set can begin with the
+ // tail of a poison cluster -- the fan-out stopped inside it, so the
+ // rest of it is the first thing here -- and a streak would stop again
+ // in the same place and strand everything after.
+ //
+ // The budget is the limit itself, not twice it. Twice was chosen to
+ // walk out of a cluster in ONE pass; `drainInPasses` walks out of it by
+ // changing direction instead. `drainFailureBudget` is shared with the
+ // tests, which previously asserted the stranding guarantee at a budget
+ // the CLI never runs at and so could not see it fail.
+ const leftover = await drainInPasses(items, async pending => {
+ commitBreaker.reset();
+ const tookBefore = patchesTheBackendTook;
+ await commitItemsSerially(pending, recordDrainMs, undefined, {
+ failureBudget: drainFailureBudget(commitBreaker.limit),
+ });
+ // Whatever the budget did not reach was pushed straight back onto the
+ // held list by the per-file loop; take it back out so the next pass
+ // owns it and the caller does not see it twice.
+ return {
+ placed: patchesTheBackendTook > tookBefore,
+ unreached: deferredByCutoff.splice(0, deferredByCutoff.length),
+ };
+ });
+ // A loop, not a spread. The ordinary path bounds this at
+ // `PARSE_STREAM_CHUNK` (500), but the reconcile call passes every
+ // deletion patch in the run unchunked, so the argument limit is not
+ // obviously out of reach -- and spreading past it throws RangeError from
+ // inside the commit phase, an uncaught throw that loses the whole
+ // batch's error accounting. The loop costs nothing and removes the
+ // question.
+ for (const item of leftover) deferredByCutoff.push(item);
+ };
+
+ const runChunk = async (ci: number): Promise<void> => {
+ const chunk = chunks[ci];
+ const endFile = chunk[chunk.length - 1].fileNumber;
+ const commitIndividually = async (
+ items: PreparedPatch[],
+ bulkError?: unknown,
+ fallbackOpts?: { replay?: boolean },
+ ): Promise<void> =>
+ commitItemsSerially(items, ms => { commitMsPerChunk[ci] += ms; }, bulkError, fallbackOpts);
const hasDeletion = chunk.some(item => patchRequiresPerFileCommit(item.patch));
@@ -1587,9 +2184,32 @@ export async function ingestFiles(
onBulkCommitted: (items, result) => {
latestRev = advanceRev(latestRev, result.rev);
patchesApplied += items.length;
+ patchesTheBackendTook += items.length;
+ commitBreaker.recordSuccess();
for (const item of items) opts?.onCommitted?.(item, result.rev);
},
commitIndividually,
+ shouldStop: () => commitBreaker.tripped(),
+ onAbandoned: (items, err) => {
+ // The clock's, if it has fired. `shouldStop` is the breaker, which
+ // does not consult the deadline, so a chunk whose bulk was aborted
+ // by the run budget arrived here and was reported as withheld by
+ // the cutoff -- the same misattribution the two lists exist to
+ // keep apart, on the one path that was not routing through them.
+ const held = runDeadlineExpired() ? deferredByDeadline : deferredByCutoff;
+ for (const item of items) held.push(item);
+ // Keep the quoted error current. This is the only place a bulk
+ // failure is seen once the breaker has tripped -- the per-file
+ // loop it would otherwise reach is exactly what the trip skips --
+ // and after the first batch trips, every later batch fails here
+ // and nowhere else. Without it the banner quotes batch 1's error
+ // for the whole run, which is wrong in the case the message was
+ // written for: a backend that recovers from saturation and starts
+ // REJECTING patches instead still reads as "the database is busy".
+ if (err !== undefined && commitFailureIndictsBackend(err, runDeadlineExpired())) {
+ commitBreaker.noteError(err);
+ }
+ },
patchIdOf: item => item.patch.patchId,
onPartialBulk: (landed, missing, err) => {
if (!debug) return;
@@ -1625,7 +2245,79 @@ export async function ingestFiles(
Array.from({ length: Math.min(COMMIT_CONCURRENCY, chunks.length) }, () => worker())
);
- return commitMsPerChunk.reduce((a, b) => a + b, 0);
+ // Nothing stays abandoned while the backend is demonstrably accepting
+ // writes. Five adjacent patches the backend rejects on their own merits
+ // look exactly like a dead backend until a later chunk commits, so the
+ // held-back set gets a real attempt rather than being written off.
+ //
+ // `drainInPasses` is what decides when to stop -- a pass that REACHED
+ // THE END, or three passes (tail, head, middle) that between them placed
+ // nothing at all, and not even that once the backend has proved it is
+ // alive. What the last pass could not reach is left for the next run,
+ // which re-ingests it anyway because the mtime baseline is not written on
+ // a run with commit errors. (A single pass that placed nothing is
+ // deliberately NOT a stopping condition, and there is no pass cap; its own
+ // doc says why for both.) The gate below is a coarser judgement across
+ // BATCHES, so that a genuinely dead backend is not re-probed once per
+ // batch for the length of the run.
+ if (deferredByCutoff.length > 0) {
+ if (!runDeadlineExpired() && drainGate.shouldDrain(patchesTheBackendTook)) {
+ const held = deferredByCutoff.splice(0, deferredByCutoff.length);
+ const tookBefore = patchesTheBackendTook;
+ const indictedBefore = backendIndictingFailures;
+ await commitIndividuallyAfterCutoff(held);
+ // A miss means the backend took nothing AND said something that
+ // indicts it. Both halves are load-bearing, and each was wrong on its
+ // own in an earlier revision:
+ //
+ // - taking nothing alone is not it. `commitFailureIndictsBackend`
+ // excludes payload-too-large, so a held set of oversized generated
+ // files is attempted in full and refused in full without the
+ // backend being at fault; two such batches closed the gate on a
+ // backend that never refused a write.
+ // - "nothing was stranded" alone is not it either, which is subtler.
+ // A held set no larger than one pass's budget -- a trip near the
+ // end of a chunk, or the deletion path where each deletion is its
+ // own one-patch chunk -- is attempted in full and refused in full
+ // by a DEAD backend, and reporting that as progress meant the gate
+ // could never close and every batch re-probed, which is the whole
+ // thing the gate exists to stop.
+ //
+ // Whether the drain ended tripped says nothing useful either: one
+ // that commits hundreds and then meets the tail of a bad cluster ends
+ // tripped too.
+ const backendMisbehaved = backendIndictingFailures > indictedBefore;
+ drainGate.record(patchesTheBackendTook > tookBefore || !backendMisbehaved, patchesTheBackendTook);
+ }
+ // Held back again, or held back after the retry was already spent.
+ if (deferredByCutoff.length > 0) {
+ commitErrors += deferredByCutoff.length;
+ // Counted as the cutoff's even when the deadline has since fired.
+ // Everything on this list was held by `perFileAction` BEFORE the
+ // clock ran out -- the deadline check sits above the hold branch and
+ // diverts to `deferredByDeadline` once it has -- so they are the
+ // cutoff's patches, and suppressing the count reported "0 of them
+ // were withheld by this cutoff" while the deadline message claimed
+ // all of them. That is the same misattribution the
+ // `deferredByDeadline` split exists to prevent, pointing the other
+ // way. The contradiction it was guarding against is handled where it
+ // belongs: `describeCommitCutoff` drops the "would have added load"
+ // framing when `deadlineHit`, and names the split instead.
+ commitBreaker.recordSkipped(deferredByCutoff.length);
+ deferredByCutoff.length = 0;
+ }
+ }
+
+ // Missing from the graph either way, but attributed to the clock: the
+ // deadline branch of `describeCommitOutcome` is what reports these.
+ if (deferredByDeadline.length > 0) {
+ commitErrors += deferredByDeadline.length;
+ // Deliberately not fed to the cutoff's own accounting: these are the
+ // clock's, and `describeCommitOutcome` reports them under the deadline.
+ deferredByDeadline.length = 0;
+ }
+
+ return commitMsPerChunk.reduce((a, b) => a + b, 0) + drainMs;
};
/** Resolve edges within a batch, build patches, and commit in sub-chunks. */
@@ -2224,7 +2916,11 @@ export async function ingestFiles(
patchesApplied,
// ix map has no --debug and never passes one; --verbose is its equivalent.
opts.mapMode === true ? "--verbose" : "--debug",
- opts.deadlineSignal?.aborted === true
+ opts.deadlineSignal?.aborted === true,
+ commitBreaker.skipped(),
+ // The same expression the banner is gated on, so this can never say "See
+ // the cutoff above" with no cutoff above it.
+ commitBreaker.everTripped()
);
const summary: IngestFilesSummary = {
filesDiscovered,
@@ -2233,6 +2929,28 @@ export async function ingestFiles(
commitErrors,
stitchErrors,
};
+ // `everTripped()` alone, and it is not a weakening: `recordFailure` has one
+ // call site, in the per-file catch immediately after `commitErrors++`, so a
+ // trip already implies a real commit failure. A `commitLoopErrors` counter
+ // guarded this for two rounds against a scenario that no longer exists --
+ // replay failures were moved off `recordFailure` onto a local counter, and a
+ // deletion patch that fails to BUILD never reaches the breaker at all. A dead
+ // conjunct defended by a comment describing an impossible case is the exact
+ // shape this PR has spent rounds removing.
+ if (commitBreaker.everTripped()) {
+ // Gated on the cutoff having FIRED, not on what was left unsent. A patch
+ // that was never sent produces no `[commit error]` line even under
+ // --verbose, so this has to print for those -- but gating on them alone
+ // meant a run whose drain eventually placed every held patch printed
+ // nothing at all. The 12-file reproduction at the default limit is exactly
+ // that shape: it held 7, the drain sent all 7, and the user got the generic
+ // "all 12 patches failed to commit, re-run with --verbose" -- the
+ // unexplained failure this change exists to remove.
+ //
+ // Before the stitch line and before the commit report: this is the cause,
+ // and everything else printed about this run is its consequence.
+ process.stderr.write(`${describeCommitCutoff(commitBreaker, client.endpoint, patchesApplied, opts.deadlineSignal?.aborted === true)}\n`);
+ }
if (stitchErrors > 0) {
process.stderr.write(` ${describeStitchFailure(stitchError)}\n`);
process.exitCode = 1;
diff --git a/ix-cli/src/cli/commit-breaker.ts b/ix-cli/src/cli/commit-breaker.ts
new file mode 100644
index 00000000..2cce2c7b
--- /dev/null
+++ b/ix-cli/src/cli/commit-breaker.ts
@@ -0,0 +1,535 @@
+// ---------------------------------------------------------------------------
+// Run-scoped cutoff for an ingest whose every commit is failing.
+//
+// Background (Ix#560): a bulk commit that fails for a reason specific to the
+// GROUP -- too large, partially applied -- is recoverable by sending the
+// patches differently, and `commitBulkWithPayloadSplit` does exactly that: it
+// falls back to committing each patch on its own. That is right for a group
+// problem and exactly wrong for a BACKEND problem. When ArangoDB cannot begin a
+// transaction at all, the fallback turns one failed request into one failed
+// request per patch, serialized behind the fallback mutex, each waiting out its
+// own timeout.
+//
+// Measured against a backend that fails every commit, on a 21-patch ingest:
+//
+// defaults (batch 1000, concurrency 8) 1 bulk + 21 per-file = 22 requests
+// IX_COMMIT_HTTP_MAX_FILES=1,
+// IX_COMMIT_CONCURRENCY=1 21 bulk + 21 per-file = 42 requests
+//
+// which is why #560 reports that lowering batch size and concurrency does not
+// help: the fan-out is per patch, so it ignores both. On a first map of a large
+// repo the same shape sends thousands of doomed requests to a backend that is
+// already the reason they are failing.
+//
+// The rule here is deliberately about BEHAVIOUR, not about error text. Which
+// message ArangoDB, the memory layer or an intervening proxy produces for
+// "saturated" varies by deployment, and a classifier assembled from guessed
+// strings is inert in exactly the deployments it was written for. What is
+// unambiguous is a backend that has refused N commits in a row without
+// accepting one: nothing about sending the next request differs from the last N.
+//
+// What the cutoff does NOT do is abandon work permanently. Patches it holds
+// back are re-sent through the same per-file loop, in passes that change
+// direction -- see `drainInPasses`, which owns the rule and its bound. The
+// breaker's STREAK is not that bound and never was: `perFileAction` ignores
+// `tripped` whenever a failure budget is passed, and the drain always passes
+// one. What bounds it is the budget per pass and the three-sample stopping
+// rule. The give-up clears only from evidence inside the same drain -- a
+// pass that places something resets the counter -- and not from the backend
+// having been alive earlier in the run, which is stale by the time a trip
+// happens.
+//
+// The streak is still reset per pass, for a different reason: five adjacent
+// patches the backend rejects on their own merits look exactly like a dead
+// backend until the next patch commits. Without any of this the mtime baseline
+// (never written on a run with commit errors) guarantees the next `ix map`
+// re-ingests in the same order, stops at the same point, and drops the same
+// patches forever.
+//
+// It also never skips a chunk's BULK commit. One request per chunk is not the
+// amplification -- the fan-out is, at one request per patch -- and sending it is
+// what lets a backend that has recovered mid-run prove so and unstick the rest.
+// ---------------------------------------------------------------------------
+
+const DEFAULT_LIMIT = 5;
+
+/** Consecutive failed commits before a run stops trying. 0 disables the cutoff. */
+export function commitFailureLimit(raw = process.env.IX_COMMIT_FAILURE_LIMIT): number {
+ if (raw === undefined || raw === "") return DEFAULT_LIMIT;
+ // Matched, not coerced. `parseInt` stopped at the first non-digit, reading
+ // "0.5" as 0 -- silently DISABLING the cutoff, the opposite of a
+ // conservative fallback -- and "1e3" as 1, tripping on the very first
+ // failure. `Number` fixes those but accepts its own surprises: whitespace
+ // coerces to 0 (disabling it again) and "0x10" to 16. A plain-decimal match
+ // accepts exactly what the docs describe and falls back on everything else.
+ if (!/^\d+$/.test(raw.trim()) || raw.trim() !== raw) return DEFAULT_LIMIT;
+ const n = Number(raw);
+ // A plain-decimal value too large to be exact is still an unambiguous
+ // instruction: "effectively never trip". Falling back to the default gave
+ // them the TIGHTEST cutoff instead -- the opposite of what they asked for,
+ // and the one direction where guessing wrong loses writes.
+ if (!Number.isSafeInteger(n)) return Number.MAX_SAFE_INTEGER;
+ return n;
+}
+
+export interface CommitBreaker {
+ /** Consecutive failures required to trip. 0 means this breaker never trips. */
+ readonly limit: number;
+ /**
+ * True while the run has given up on the backend.
+ *
+ * Sticky against further FAILURES -- a streak that has already tripped stays
+ * tripped -- but a single success clears it. A commit landing is direct
+ * evidence the backend accepts writes, and there is nothing left to give up
+ * about.
+ *
+ * An earlier revision latched permanently, to stop a 413 split's second half
+ * succeeding and making a run that had abandoned patches claim it never gave
+ * up. Patches are no longer abandoned on trip -- they are held and re-sent --
+ * so that contradiction is gone, and the permanent latch had become the
+ * worse bug: after one trip, a backend that recovered completely still had
+ * every deletion patch dropped for the rest of the run, because those never
+ * go through the bulk path that would have proved it healthy.
+ */
+ tripped(): boolean;
+ /** A commit landed. The backend is accepting writes, so the run resumes. */
+ recordSuccess(): void;
+ /** A commit failed in a way that sending it again cannot fix. */
+ recordFailure(error: unknown): void;
+ /**
+ * A failure worth QUOTING, but not worth counting.
+ *
+ * A bulk commit that fails while the breaker is already tripped is abandoned
+ * without any per-file attempt, so nothing else here ever sees it -- and
+ * that is the dominant shape of a dead-backend run, since a batch is one
+ * chunk. Without this, `lastError()` freezes at the last per-file failure
+ * and the banner keeps quoting it for the rest of the run: five
+ * `TimeoutError`s in batch 1, then nineteen batches rejected with
+ * `Invalid message body`, and the user is still told the database is busy
+ * and sent to `docker stats`. That is the misrouting the message exists to
+ * prevent, so the quote has to keep up.
+ *
+ * Deliberately not `recordFailure`: one bulk is one request, not N, and
+ * counting it would let a single abandoned bulk stand in for a streak.
+ */
+ noteError(error: unknown): void;
+ /**
+ * Did the run give up on the backend at ANY point?
+ *
+ * Distinct from `tripped()`, which a later success clears, and from
+ * `skipped()`, which counts only what was still unsent at the end. The
+ * diagnosis is gated on this: a run whose drain eventually placed every held
+ * patch has `skipped() === 0`, and gating the banner on that meant the one
+ * message #560 asked for -- the reason `ix doctor` passes while nothing
+ * commits -- was printed only when the run ALSO lost patches. A 12-file
+ * ingest against a locked ArangoDB at the default limit hit exactly that: it
+ * held 7, the drain sent all 7, and the user got the generic "all N patches
+ * failed to commit, re-run with --verbose" and no diagnosis at all.
+ */
+ everTripped(): boolean;
+ /** Patches abandoned because the breaker was already tripped. */
+ skipped(): number;
+ /** Count one abandoned patch. */
+ recordSkipped(n?: number): void;
+ /** The most recent failure, for the message. */
+ lastError(): unknown;
+ /** Failures in the current streak. */
+ consecutiveFailures(): number;
+ /**
+ * Start a fresh streak, un-latched.
+ *
+ * For the drain: called once per pass, in every batch that holds patches.
+ * They are re-sent with no memory of the streak that held them, because five
+ * adjacent patches the backend rejects on their own merits look exactly like
+ * a dead backend until the next one commits.
+ *
+ * What it does NOT do is bound the drain. `perFileAction` ignores `tripped`
+ * whenever a failure budget is passed and the drain always passes one, so the
+ * bound is `drainInPasses`'s, not this. `skipped()` is cumulative and
+ * deliberately not reset.
+ */
+ reset(): void;
+}
+
+export function createCommitBreaker(limit = commitFailureLimit()): CommitBreaker {
+ let consecutive = 0;
+ let latched = false;
+ let everLatched = false;
+ let skippedCount = 0;
+ let last: unknown;
+
+ return {
+ limit,
+ tripped: () => latched,
+ everTripped: () => everLatched,
+ recordSuccess: () => { consecutive = 0; latched = false; },
+ noteError: (error) => { last = error; },
+ recordFailure: (error) => {
+ consecutive++;
+ last = error;
+ if (limit > 0 && consecutive >= limit) {
+ latched = true;
+ everLatched = true;
+ }
+ },
+ skipped: () => skippedCount,
+ recordSkipped: (n = 1) => { skippedCount += n; },
+ lastError: () => last,
+ consecutiveFailures: () => consecutive,
+ reset: () => { consecutive = 0; latched = false; },
+ };
+}
+
+/** What the per-file loop should do with the next patch. */
+export type PerFileAction =
+ /** Send it. */
+ | "send"
+ /** Do not send it; hold it for the end-of-run drain. */
+ | "hold"
+ /** Do not send it; it is already in the graph. Count it and stop the loop. */
+ | "count-applied";
+
+/**
+ * The whole of the per-file loop's stop/hold decision, in one place.
+ *
+ * Extracted because it lives inside `ingestFiles` otherwise, closing over a
+ * dozen locals, and every revision of it that could not be driven directly grew
+ * a defect that only a hand-built end-to-end run would show: a budget that
+ * never bound because the streak preempted it, a replay guard whose condition
+ * had been inverted, an entry check that turned a bulk into N serialized
+ * commits. Those are all decision-table bugs, and a decision table should be
+ * readable and assertable on its own.
+ *
+ * `budgetLeft` and `tripped` are two different ways of saying "stop", and which
+ * one applies is the substance:
+ *
+ * - A caller with a BUDGET has said it wants to be bounded by the arithmetic
+ * of total failures, not by adjacency. The drain is that caller: the set it
+ * is re-sending begins with whatever run of bad patches stopped the fan-out,
+ * so a streak would stop again in the same place and strand everything
+ * behind it -- the permanent wedge, since the mtime baseline is never
+ * written on a run with commit errors and the next run repeats the order.
+ * - A caller with no budget uses the breaker's consecutive streak, which is
+ * the right bound for a fan-out that has a drain behind it to catch what it
+ * holds.
+ *
+ * A REPLAY is neither: those patches are confirmed landed by the server's own
+ * 409 body, so re-sending is bookkeeping and failing to is not a lost write.
+ */
+export function perFileAction(state: {
+ /** Is the caller re-sending patches the server has confirmed it holds? */
+ replay: boolean;
+ /** Has the run-wide breaker given up? */
+ tripped: boolean;
+ /** Failures this loop may still absorb, or undefined if it has no budget. */
+ budgetLeft?: number;
+}): PerFileAction {
+ const outOfBudget = state.budgetLeft !== undefined && state.budgetLeft <= 0;
+ // A replay is decided FIRST, and is never held. Both reasons to stop want the
+ // same thing from it -- stop sending -- but a hold is not that: the caller
+ // turns everything it holds into a commit error, so holding a replay would
+ // report writes the server has already CONFIRMED as failures and suppress the
+ // mtime baseline over them. Counting stops the fan-out just as dead and says
+ // the true thing. (An earlier revision returned "hold" here; no caller passes
+ // both a replay and a budget today, so it was a latent contradiction between
+ // this table and its only consumer rather than a live bug.)
+ if (state.replay) return state.tripped || outOfBudget ? "count-applied" : "send";
+ if (outOfBudget) return "hold";
+ // Only when no budget was given -- with both live the streak always preempts,
+ // which is exactly how the budget came to be inert.
+ if (state.budgetLeft === undefined && state.tripped) return "hold";
+ return "send";
+}
+
+/** Failures one drain pass absorbs before it gives up and hands back the rest. */
+export function drainFailureBudget(limit = commitFailureLimit()): number {
+ return Math.max(1, limit);
+}
+
+/** What one pass of the drain achieved. */
+export interface DrainPass<T> {
+ /** Did the backend accept at least one patch during this pass? */
+ placed: boolean;
+ /** Items the pass never attempted, in the order it would have taken them. */
+ unreached: T[];
+}
+
+/**
+ * Walk a held-back set, changing direction each time the budget stops it.
+ *
+ * The guarantee: **a patch is only ever handed back unsent once three passes --
+ * from the tail, the head and the middle -- have between them placed nothing at
+ * all.** Everything else is attempted.
+ *
+ * Two rules get there, and each exists because the other one alone is wrong.
+ *
+ * A pass stops after `budget` indicting failures, so it can end anywhere and
+ * hand back everything past that point -- and handed back means reported as a
+ * commit error, which suppresses the mtime baseline, which makes the next
+ * `ix map` repeat the same order and strand the same patches forever. So the
+ * next pass approaches the region it could not reach from the OTHER end, which
+ * is what stops one cluster of bad patches from hiding the good ones behind it.
+ *
+ * What ends it is THREE passes -- tail, head, middle -- that between them place
+ * nothing at all. Each weaker rule strands committable patches, and each one
+ * shipped here before the next was found:
+ *
+ * - "a pass placed nothing" is not evidence. A pass that spent its whole
+ * budget inside a leading cluster has said nothing about the region it
+ * never reached, and this stranded 400 committable patches.
+ * - "two passes in a row placed nothing" is not evidence either. Held
+ * `[b x 5, g x 400, b x 10, h x 485]`: the first pass places 485 and stops
+ * inside the second cluster, and the next two each spend their budget on a
+ * cluster now at the start of their direction -- 400 good patches between
+ * them handed back on a backend that had just taken 485 writes.
+ * - "two passes, nothing placed in the whole drain" is not evidence EITHER,
+ * and this is the one that is genuinely hard to see. Held
+ * `[b x 5, g x 400, B x 10]`: pass one walks from the tail into `B`, pass
+ * two from the head into `b`, both spend their whole budget without
+ * placing, and the 400 between are stranded. Two passes only ever sample
+ * the two ENDS.
+ *
+ * So the third pass starts in the MIDDLE. Three samples -- both ends and the
+ * centre -- all placing nothing is the bar, and one patch landing anywhere
+ * clears it: after that the failures are about the patches, and every remaining
+ * one deserves its turn.
+ *
+ * The residue is stated rather than hidden: a held set with `budget` consecutive
+ * indicting failures at the head, the tail AND the midpoint still gives up with
+ * work left. That needs fifteen bad patches in three specific places, and the
+ * alternative -- never giving up -- costs the whole held set on a dead backend,
+ * which is what `main` already does and what #560 is about.
+ *
+ * A `mayGiveUp` flag briefly narrowed that further, driven by whether the batch
+ * had accepted anything before the trip. It is gone, because the evidence is
+ * from the wrong moment: the successes come BEFORE the degradation in the
+ * normal ordering -- the breaker trips after them -- so one patch landing early
+ * in a 500-patch batch disabled the stopping rule entirely and the drain then
+ * issued up to 494 serialized commits against a dead backend, each with its own
+ * conflict retries. That is #560's amplification in full, reachable from a
+ * single lucky patch, and it is a worse failure than the residue it closed.
+ *
+ * The evidence that is not stale is this function's own: `emptyPasses` resets
+ * whenever a pass places something, so a backend that is actually taking writes
+ * already prevents the give-up for as long as it keeps taking them.
+ *
+ * What this deliberately does NOT have is a pass cap. A cap sounds like the
+ * safe choice and is the opposite: it bounds the walk at `passes x budget`
+ * failing patches -- 15 at the shipped numbers -- and hands back every
+ * committable patch beyond that unsent, which is a net loss against `main` on a
+ * backend that is accepting writes. Measured on the earlier capped version: a
+ * 1,000-patch held set with 20 scattered bad patches placed 686 and stranded
+ * 294. Termination does not need a cap, because a pass that continues has
+ * attempted at least `budget` items, so `unreached` strictly shrinks.
+ *
+ * The cost has two shapes, and the second one is the price of the guarantee.
+ * A backend that accepts NOTHING is stopped after `3 x budget` doomed requests
+ * -- one per sample -- which is 15 at the shipped default.
+ * A backend that accepts something and then refuses the rest is not stopped at
+ * all: each pass eats `budget` of the refusing patches, so a 995-patch held set
+ * where one write lands and 994 are refused runs ~199 passes and sends all 995
+ * requests.
+ *
+ * That is deliberate, and it is not a regression -- 995 requests is exactly
+ * what `main` sends for 995 patches, one per patch, with no cutoff at all. The
+ * total is bounded by the held-set size in every case, because a pass only ever
+ * attempts items no earlier pass reached. What the cutoff buys on that backend
+ * is the FAN-OUT stopping after `limit`, and `createDrainGate` stopping the
+ * drain repeating once per batch. Bounding it further is not available: any cap
+ * on failures hands back committable patches unsent, which is a real loss where
+ * this is only slowness.
+ */
+export async function drainInPasses<T>(
+ held: readonly T[],
+ attempt: (items: T[]) => Promise<DrainPass<T>>,
+): Promise<T[]> {
+ /** Start the walk at the midpoint, so a pass samples neither end. */
+ const fromTheMiddle = <U,>(items: U[]): U[] => {
+ const half = items.length >> 1;
+ return [...items.slice(half), ...items.slice(0, half)];
+ };
+
+ let pending = [...held].reverse();
+ let emptyPasses = 0;
+ while (pending.length > 0) {
+ const { placed, unreached } = await attempt(pending);
+ if (unreached.length === 0) return [];
+ emptyPasses = placed ? 0 : emptyPasses + 1;
+ // Tail, head and middle have all placed nothing: the backend, not the
+ // patches. Fewer samples than that only ever cover the ends.
+ if (emptyPasses >= 3) return unreached;
+ pending = [...unreached].reverse();
+ // The third attempt is the one that has to avoid both ends.
+ if (emptyPasses === 2) pending = fromTheMiddle(pending);
+ }
+ return [];
+}
+
+/** Whether a batch's held-back patches are worth draining. See [[createDrainGate]]. */
+export interface DrainGate {
+ /** Is the gate open right now? Pass the run's applied count. */
+ shouldDrain(appliedSoFar: number): boolean;
+ /** Record what a drain achieved, so the gate can close on a dead backend. */
+ record(placedAnything: boolean, appliedSoFar: number): void;
+}
+
+/**
+ * The drain decision, one level up from `drainInPasses`: per BATCH, not per pass.
+ *
+ * A run has many batches, and each can hold patches back. Without a gate a
+ * genuinely dead backend is re-probed once per batch for the length of the run
+ * -- forty batches, each spending a full drain budget of doomed requests --
+ * which is the amplification #560 is about, moved rather than removed. So after
+ * `missesBeforeGivingUp` drains that place NOTHING, the gate closes.
+ *
+ * It reopens the moment a patch lands anywhere -- and, failing that, lets one
+ * drain through every `probeEveryBatches`, because a closed gate can block the
+ * progress its own reopen condition waits for. That direction is the one that
+ * matters and the one an earlier revision got wrong: closing is an INFERENCE
+ * about the backend drawn from two drains, and the run keeps going afterwards.
+ * A thirty-second blip in batches 3 and 4 must not disable the drain for the
+ * remaining thirty-five, because a hold at batch 30 -- against a backend that
+ * has since committed nineteen thousand patches -- would then have its patches
+ * counted as errors and never sent. Anything landing after the gate closed
+ * falsifies the evidence that closed it.
+ *
+ * One miss is not enough evidence to close on: a few-second fast-rejecting blip
+ * produces it, and closing then strands a later batch's held patches with no
+ * drain at all.
+ */
+export function createDrainGate(missesBeforeGivingUp = 2, probeEveryBatches = 5): DrainGate {
+ let misses = 0;
+ let closed = false;
+ let appliedWhenClosed = 0;
+ let batchesSinceClosed = 0;
+ return {
+ shouldDrain(appliedSoFar) {
+ if (!closed) return true;
+ if (appliedSoFar > appliedWhenClosed) {
+ closed = false;
+ misses = 0;
+ return true;
+ }
+ // A patch landing is the reopen condition, and a closed gate can prevent
+ // the very progress it waits for: with the drain off, held patches go
+ // straight to commit errors without a request, so the per-file and
+ // drain-probe sources of `appliedSoFar` are gone and only a BULK success
+ // is left. A backend that recovers while its bulks keep failing for GROUP
+ // reasons -- an old memory layer answering "partially committed" with no
+ // parseable ids, or a tail of oversized files bisecting to single-item
+ // 413s -- can then never clear the breaker or the gate, and every
+ // remaining patch in the run is reported as an error with nothing sent.
+ // `main` would have committed them per file.
+ //
+ // So the gate lets one drain through every `probeEveryBatches`. It is the
+ // same trade the cutoff makes everywhere else: a bounded number of
+ // requests against the possibility that the run is wrong about the
+ // backend.
+ batchesSinceClosed++;
+ if (batchesSinceClosed >= probeEveryBatches) {
+ batchesSinceClosed = 0;
+ return true;
+ }
+ return false;
+ },
+ record(placedAnything, appliedSoFar) {
+ misses = placedAnything ? 0 : misses + 1;
+ if (misses >= missesBeforeGivingUp) {
+ closed = true;
+ appliedWhenClosed = appliedSoFar;
+ batchesSinceClosed = 0;
+ }
+ },
+ };
+}
+
+/**
+ * What the user is told when a run stops early.
+ *
+ * This is the diagnostic #560 did not have. The symptom there is an ingest that
+ * fails while `ix doctor` reports a healthy graph -- doctor asks whether the
+ * backend is reachable and the graph is consistent, neither of which is false
+ * when ArangoDB is merely too busy to start a transaction. Saying plainly that
+ * the backend refused every write, and quoting the error once instead of once
+ * per file behind --debug, is the difference between "Ix is broken" and "the
+ * database is saturated".
+ */
+export function describeCommitCutoff(
+ breaker: CommitBreaker,
+ endpoint: string,
+ /** Patches that landed anywhere in the run. */
+ patchesApplied = 0,
+ /**
+ * Did the run's own wall-clock budget fire?
+ *
+ * When it did, the deadline branch of `describeCommitOutcome` owns what to do
+ * about the missing patches, and the two must not contradict each other: a
+ * cutoff that fired BEFORE the clock has already called `recordSkipped`, so
+ * this would otherwise print "N were not sent, sending them would have added
+ * load to a backend that is already the reason they fail" directly above
+ * "Ingest ran out of time: raise IX_MAP_DEADLINE_MS". The diagnosis below --
+ * what the backend actually answered -- is still worth printing; the
+ * attribution of the count is not.
+ */
+ deadlineHit = false,
+): string {
+ const detail = String(breaker.lastError() ?? "unknown error");
+ const trimmed = detail.length > 300 ? `${detail.slice(0, 300)}…` : detail;
+ // A backend can start refusing part-way through a large ingest, so "the
+ // graph is unchanged" is only true when nothing landed. Saying it anyway
+ // contradicts the summary printed directly beneath this.
+ //
+ // Deliberately NOT "landed BEFORE it stopped". Since a cutoff holds patches
+ // and the run continues, there is no single moment it stopped: a run that
+ // tripped in batch 1, recovered, and committed 19,000 more would have claimed
+ // all 19,000 landed "before it stopped". Naming no order needs no snapshot to
+ // be true, which is also one less piece of run-scoped state to keep correct.
+ const state =
+ patchesApplied === 0
+ ? "The graph is unchanged."
+ : `${patchesApplied} other ${patchesApplied === 1 ? "patch" : "patches"} did land, so the graph is partly updated.`;
+ // No failure count in this message at all, deliberately, and neither
+ // `breaker.limit` nor the live streak is read here. The fan-out stops on a
+ // streak, the drain on a per-pass budget, and the batch gate on two empty
+ // drains -- this banner prints for all three, so any single number it named
+ // would be wrong for two of them. Earlier revisions named one and had to
+ // keep re-explaining which; saying "kept failing" is true of every path.
+ // "Commits kept failing", not "Stopped committing": this prints whenever the
+ // run gave up at any point, and the drain may since have placed every patch
+ // it held. Claiming it stopped would then be false on the one line whose job
+ // is saying what happened.
+ const n = breaker.skipped();
+ const unsent =
+ n === 0
+ ? []
+ : deadlineHit
+ // The deadline branch of `describeCommitOutcome` reports the whole
+ // commit-error total as abandoned by the clock, and it is printed right
+ // after this. Saying "sending them would have added load to a backend
+ // that is already the reason they fail" beside it is two answers to one
+ // question -- but dropping the line entirely handed the cutoff's share
+ // to the clock and pointed the reader at IX_MAP_DEADLINE_MS for patches
+ // deliberately withheld from a refusing backend. Name the split.
+ ? [` ${n} of them ${n === 1 ? "was" : "were"} withheld by this cutoff before the run ran out of time.`]
+ : [
+ ` ${n} ${n === 1 ? "patch was" : "patches were"} not sent — sending them one at a`,
+ ` time would have added load to a backend that is already the reason they fail.`,
+ ];
+ return [
+ `Error: Commits against ${endpoint} kept failing, so ix stopped fanning out one request per patch.`,
+ ...unsent,
+ ` Last error: ${trimmed}`,
+ ` ${state} Re-run \`ix map\` once it is fixed.`,
+ // Two different causes produce this, and the CLI cannot tell them apart:
+ // the memory layer answers 500 both for a saturated ArangoDB and for a
+ // patch body it will not accept (verified against the released image).
+ // Asserting the database is the cause sends a user with a rejected patch
+ // to `docker stats` and `/_api/query/current`, which is the wrong
+ // subsystem entirely. The backend's own words are quoted above; let them
+ // say which.
+ ` Either the database cannot keep up — \`ix doctor\` still passes, since an ArangoDB that`,
+ ` cannot begin a transaction is reachable and consistent — or the backend is rejecting these`,
+ ` particular patches. The error above is the backend's own answer and says which.`,
+ ` Set IX_COMMIT_FAILURE_LIMIT=0 to send every patch regardless.`,
+ ].join("\n");
+}
diff --git a/skills/ix/references/troubleshooting.md b/skills/ix/references/troubleshooting.md
index fc06dbff..dc179ab1 100644
--- a/skills/ix/references/troubleshooting.md
+++ b/skills/ix/references/troubleshooting.md
@@ -38,6 +38,7 @@ again.
| `Compass UI not found` on `ix view` | Run `ix upgrade` to fetch Compass. Re-running the installer wipes the Compass assets, so bootstrap auto-restores them with `ix upgrade`; skip with `IX_SKIP_COMPASS=1` |
| Compass shows "Compass not connected to a codebase" | The scoped workspace has no graph — run `ix map .` from the repo root, then reload the tab |
| Slow or stale results | Re-run `ix map --silent` to refresh the graph |
+| `Commits against <endpoint> kept failing` | Commits kept failing, so `ix map` gave up rather than sending one doomed request per file. Read the `Last error` line it prints — the CLI cannot tell the two causes apart, because the memory layer answers 500 for both. **A saturated database:** `ix doctor` still passes (it asks whether the backend is reachable and the graph consistent, and an ArangoDB too busy to begin a transaction is both), so check `docker stats` and `GET /_db/<db>/_api/query/current`, then re-map once it is idle. **Patches the backend will not accept:** the error names the patch or the field; re-map after fixing it, or `IX_COMMIT_FAILURE_LIMIT=0` to send the rest regardless. |
## Environment flags
@@ -45,6 +46,9 @@ again.
- `IX_SKIP_INSTALL=1` — skip the CLI install step in bootstrap.
- `IX_SKIP_BACKEND=1` — skip starting/waiting for the backend in bootstrap.
- `IX_SKIP_MAP=1` — skip `ix map` in bootstrap.
+- `IX_COMMIT_FAILURE_LIMIT=N` — consecutive failed commits before `ix map`
+ stops sending more (default 5). `0` sends every patch regardless, which is
+ the pre-#560 behaviour.
## Re-mapping