Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

scale: import_rate sets aside one stall, never recurring work (#7939)

garrytan077de53a2a88
diff --git a/scripts/scale/gates.ts b/scripts/scale/gates.ts
index 1c505515dc..85767a8061 100644
--- a/scripts/scale/gates.ts
+++ b/scripts/scale/gates.ts
@@ -34,7 +34,10 @@ export const RATE_RATIO_MAX = 1.5;
export const RATE_MIN_PAGES = 1000;
export const TOTAL_VS_HALF_MAX = 2.5;
-/** One timing basis of the import: per-page cost of the first and last 10%, and total vs the halfway mark. */
+/**
+ * One timing basis of the import: per-page cost of the first and last 10%, and total vs the halfway mark.
+ * The two ratios leave out the one stall `excusedStall` sets aside; `total_ms` and `ms_at_half` are the measured sums.
+ */
export interface RateMeasure {
per_page_ms_first10: number;
per_page_ms_last10: number;
@@ -42,8 +45,12 @@ export interface RateMeasure {
total_ms: number;
ms_at_half: number;
total_vs_half: number;
+ excused?: ExcusedStall;
}
+/** The milliseconds the rate measure set aside, and the 1-based pages of the slice that held them. */
+export interface ExcusedStall { ms: number; first_page: number; last_page: number }
+
/**
* Per-file wall and CPU milliseconds from `gbrain import --progress-json --progress-interval 0`
* stderr: one `import.files` tick per file, each with cumulative `elapsed_ms` and `cpu_ms`.
@@ -63,16 +70,70 @@ export function importProgressPerFile(stderr: string): { wallMs: number[]; cpuMs
return { wallMs: perFile(wall), cpuMs: perFile(cpu) };
}
+/**
+ * #7939: one stall of the import (a paused database container, a WAL checkpoint, a burst from another tenant of a
+ * shared CI host) lands in one page's time and once carried a whole tenth's mean past the gate on its own. Work that
+ * grows with the brain does not look like that: it recurs. So the measure sets aside at most one stall per import,
+ * and only the part of it no other part of the import repeats:
+ *
+ * - Each tenth of the import is cut into ten slices. A slice's excess is its mean per-page cost above the median
+ * slice of its tenth. The slice with the largest excess is lowered to the second-largest excess anywhere in the
+ * import, so a cost that shows up twice (two rebuilds, two stalls) is set aside only by the margin between them.
+ * - The lowered tenth never drops below the Theil-Sen line through the ten tenth means, so a cost that lands once in
+ * every tenth and grows with the brain is kept in full: those tenth means rise on that line.
+ *
+ * Nothing else changes: every other slice counts, and with no stall the measure is the plain mean of each tenth.
+ */
+export function excusedStall(perPageMs: number[]): ExcusedStall | undefined {
+ const n = perPageMs.length;
+ if (n < 100) return undefined;
+ const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);
+ const median = (xs: number[]) => {
+ const s = [...xs].sort((a, b) => a - b);
+ const mid = s.length >> 1;
+ return s.length % 2 ? s[mid]! : (s[mid - 1]! + s[mid]!) / 2;
+ };
+ const tenParts = (from: number, to: number) => Array.from({ length: 10 }, (_, k) => {
+ const lo = from + Math.floor((to - from) * k / 10);
+ const hi = from + Math.floor((to - from) * (k + 1) / 10);
+ return { lo, hi, mean: sum(perPageMs.slice(lo, hi)) / (hi - lo) };
+ });
+ const tenths = tenParts(0, n);
+ const slices = tenths.flatMap((tenth, d) => {
+ const parts = tenParts(tenth.lo, tenth.hi);
+ const typical = median(parts.map(s => s.mean));
+ return parts.map(s => ({ ...s, d, excess: s.mean - typical }));
+ }).sort((a, b) => b.excess - a.excess);
+ const [worst, next] = [slices[0]!, slices[1]!];
+ const means = tenths.map(t => t.mean);
+ const slopes = means.flatMap((a, i) => means.slice(i + 1).map((b, k) => (b - a) / (k + 1)));
+ const slope = median(slopes);
+ const trend = median(means.map((m, d) => m - slope * d)) + slope * worst.d;
+ const tenth = tenths[worst.d]!;
+ const pages = tenth.hi - tenth.lo;
+ const lowered = tenth.mean - (worst.excess - next.excess) * (worst.hi - worst.lo) / pages;
+ const ms = Math.round((tenth.mean - Math.min(tenth.mean, Math.max(lowered, trend))) * pages);
+ return ms > 0 ? { ms, first_page: worst.lo + 1, last_page: worst.hi } : undefined;
+}
+
export function rateMeasure(perPageMs: number[]): RateMeasure {
const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);
- const tenth = Math.max(1, Math.floor(perPageMs.length / 10));
- const first = sum(perPageMs.slice(0, tenth)) / tenth;
- const last = sum(perPageMs.slice(-tenth)) / tenth;
- const half = sum(perPageMs.slice(0, Math.floor(perPageMs.length / 2)));
- const total = sum(perPageMs);
+ const excused = excusedStall(perPageMs);
+ const judged = [...perPageMs];
+ if (excused) {
+ const stalled = judged.slice(excused.first_page - 1, excused.last_page);
+ const keep = 1 - excused.ms / sum(stalled);
+ stalled.forEach((ms, k) => { judged[excused.first_page - 1 + k] = ms * keep; });
+ }
+ const tenth = Math.max(1, Math.floor(judged.length / 10));
+ const first = sum(judged.slice(0, tenth)) / tenth;
+ const last = sum(judged.slice(-tenth)) / tenth;
+ const halfPages = Math.floor(perPageMs.length / 2);
const round = (n: number, places: number) => Math.round(n * 10 ** places) / 10 ** places;
return { per_page_ms_first10: round(first, 1), per_page_ms_last10: round(last, 1), rate_ratio: round(last / first, 2),
- total_ms: Math.round(total), ms_at_half: Math.round(half), total_vs_half: round(total / half, 2) };
+ total_ms: Math.round(sum(perPageMs)), ms_at_half: Math.round(sum(perPageMs.slice(0, halfPages))),
+ total_vs_half: round(sum(judged) / sum(judged.slice(0, halfPages)), 2),
+ ...(excused ? { excused } : {}) };
}
/**
@@ -174,10 +235,11 @@ export function evaluateScaleGates(report: ScaleReport, policy: GatePolicy): Gat
const basis = importRateBasis(report.engine);
const other = basis === 'cpu' ? 'wall' : 'cpu';
- const { rate_ratio, total_vs_half, per_page_ms_first10, per_page_ms_last10 } = report.import[basis];
+ const { rate_ratio, total_vs_half, per_page_ms_first10, per_page_ms_last10, excused } = report.import[basis];
add('import_rate', rate_ratio <= RATE_RATIO_MAX && total_vs_half <= TOTAL_VS_HALF_MAX, report.pages >= RATE_MIN_PAGES,
`import rate (${basis} time): last 10% per-page cost ${per_page_ms_last10} ms is ${rate_ratio}x the first 10% (${per_page_ms_first10} ms; gate <= ${RATE_RATIO_MAX}); `
+ `total import time is ${total_vs_half}x the time at the halfway mark (gate <= ${TOTAL_VS_HALF_MAX}). `
+ + (excused ? `Both ratios leave out one ${excused.ms} ms stall in pages ${excused.first_page}-${excused.last_page} that the rest of the import does not repeat (#7939). ` : '')
+ `Not judged on ${report.engine}: ${other} time ratio ${report.import[other].rate_ratio}, total/half ${report.import[other].total_vs_half}. `
+ `A rising per-page cost means import slows as the brain grows, usually stale planner statistics during import. Reproduce: ${repro}`);
diff --git a/test/scripts/scale-gates.test.ts b/test/scripts/scale-gates.test.ts
index 52097a4d7b..1f262578e2 100644
--- a/test/scripts/scale-gates.test.ts
+++ b/test/scripts/scale-gates.test.ts
@@ -8,7 +8,7 @@
import { describe, expect, test } from 'bun:test';
import {
CEILING_PAGES, CEILINGS_MS, ceilingsFor, evaluateScaleGates, importProgressPerFile, RATE_MIN_PAGES, HEADLINE_OP, HOT_TABLES, KEY_PLAN_OPS, phaseLimitsMs, PLANNER_HEALTH_ENFORCED, PLANNER_STATS_MIN_ROWS,
- rateMeasure, resultHits, verdictLines, type GatePolicy, type RateMeasure, type ScaleReport,
+ RATE_RATIO_MAX, rateMeasure, resultHits, TOTAL_VS_HALF_MAX, verdictLines, type GatePolicy, type RateMeasure, type ScaleReport,
} from '../../scripts/scale/gates.ts';
const ENFORCE: GatePolicy = { enforce: true, enforcePlanner: PLANNER_HEALTH_ENFORCED, enforceCeilings: false };
@@ -172,6 +172,99 @@ describe('scale gates under --enforce', () => {
});
});
+ // #7939: one stall (a paused database container, a WAL checkpoint) inside the last tenth carried its mean past the
+ // gate on unrelated PRs. The measure sets aside at most one stall, and only the part of it nothing else in the
+ // import repeats; work that recurs, which is how an import goes quadratic, keeps the plain measure's sensitivity.
+ describe('import rate sets aside one stall and never recurring work (#7939)', () => {
+ const PAGES = 10_000;
+ let state = 7939;
+ const jitter = Array.from({ length: PAGES }, () => {
+ state = (state + 0x6D2B79F5) | 0;
+ let x = Math.imul(state ^ (state >>> 15), 1 | state);
+ x = (x + Math.imul(x ^ (x >>> 7), 61 | x)) ^ x;
+ return 0.8 + 0.4 * (((x ^ (x >>> 14)) >>> 0) / 4294967296);
+ });
+ const series = (extra: (i: number) => number, noisy = true) => Array.from({ length: PAGES }, (_, i) => 5 * (noisy ? jitter[i]! : 1) + extra(i));
+ /** Master's measure before #7939: plain means of the first and last tenth, plain total over the halfway sum. */
+ const plainMeasure = (ms: number[]) => {
+ const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);
+ const round = (n: number) => Math.round(n * 100) / 100;
+ return { rate_ratio: round(sum(ms.slice(-PAGES / 10)) / sum(ms.slice(0, PAGES / 10))), total_vs_half: round(sum(ms) / sum(ms.slice(0, PAGES / 2))) };
+ };
+ const verdict = (m: { rate_ratio: number; total_vs_half: number }) => (m.rate_ratio <= RATE_RATIO_MAX && m.total_vs_half <= TOTAL_VS_HALF_MAX ? 'pass' : 'fail');
+ const stall = (page: number, ms: number) => (i: number) => (i === page ? ms : 0);
+ const rebuild = (every: number, msPerPageSoFar: number, phase = 0) => (i: number) => (i > 0 && (i - phase) % every === 0 ? msPerPageSoFar * i : 0);
+ const alternatingGrowth = (msPerPageSoFar: number) => (i: number) => (Math.floor(i / 100) % 2 ? msPerPageSoFar * i : 0);
+ const linear = (msPerPageSoFar: number) => (i: number) => msPerPageSoFar * i;
+
+ const CASES: Array<[string, 'pass' | 'fail', 'pass' | 'fail', (i: number) => number]> = [
+ ['no regression', 'pass', 'pass', () => 0],
+ ['one 12 s stall at page 9,300', 'fail', 'pass', stall(9300, 12_000)],
+ ['one 30 s stall in the last slice', 'fail', 'pass', stall(9950, 30_000)],
+ ['one 30 s stall in the second half', 'fail', 'pass', stall(7000, 30_000)],
+ ['a rebuild every 200 pages costing 0.5 ms x pages so far', 'fail', 'fail', rebuild(200, 0.5)],
+ ['the same rebuild every 150 pages', 'fail', 'fail', rebuild(150, 0.5)],
+ ['growth only on alternating 100-page blocks', 'fail', 'fail', alternatingGrowth(0.001)],
+ ['a plain O(n) per-page cost', 'fail', 'fail', linear(0.0005)],
+ ['a rebuild once per tenth (every 1,000 pages), one slice per tenth', 'fail', 'fail', rebuild(1000, 0.5)],
+ ['the same rebuild half a tenth later', 'fail', 'fail', rebuild(1000, 0.5, 500)],
+ ['two 12 s stalls in the last tenth', 'fail', 'fail', i => stall(9300, 12_000)(i) + stall(9700, 12_000)(i)],
+ ['every page of the last tenth 60% slower', 'fail', 'fail', i => (i >= 9000 ? 3 : 0)],
+ ];
+ test.each(CASES)('%s: before #7939 %s, now %s', (_name, before, now, extra) => {
+ const ms = series(extra);
+ expect(verdict(plainMeasure(ms))).toBe(before);
+ expect(verdict(rateMeasure(ms))).toBe(now);
+ });
+
+ // Equal sensitivity: the smallest amplitude that fails the gate, found by bisection, is the plain measure's to within
+ // 1% (the page-to-page jitter of the slice that is lowered; 0.8% at most here).
+ const failsAt = (measure: (ms: number[]) => { rate_ratio: number; total_vs_half: number }, family: (k: number) => (i: number) => number) => {
+ let lo = 0;
+ let hi = 1;
+ while (verdict(measure(series(family(hi)))) === 'pass') hi *= 2;
+ for (let step = 0; step < 30; step++) {
+ const mid = (lo + hi) / 2;
+ if (verdict(measure(series(family(mid)))) === 'fail') hi = mid; else lo = mid;
+ }
+ return hi;
+ };
+ const RECURRING: Array<[string, (k: number) => (i: number) => number]> = [
+ ['a rebuild every 200 pages', k => rebuild(200, k)],
+ ['a rebuild every 150 pages', k => rebuild(150, k)],
+ ['growth only on alternating 100-page blocks', k => alternatingGrowth(k)],
+ ['a plain O(n) per-page cost', k => linear(k)],
+ ['a rebuild once per tenth', k => rebuild(1000, k)],
+ ['a rebuild once per tenth, half a tenth later', k => rebuild(1000, k, 500)],
+ ];
+ test.each(RECURRING)('%s fails at the same size as under the plain measure', (_name, family) => {
+ const ratio = failsAt(rateMeasure, family) / failsAt(plainMeasure, family);
+ expect(ratio).toBeGreaterThanOrEqual(1);
+ expect(ratio).toBeLessThan(1.01);
+ });
+
+ test('a cost that grows with the brain sets nothing aside: the measure is the plain one', () => {
+ for (const extra of [linear(0.0005), rebuild(1000, 0.5), (i: number) => (i >= 9000 ? 3 : 0)]) {
+ const ms = series(extra, false);
+ expect(rateMeasure(ms).excused).toBeUndefined();
+ expect(rateMeasure(ms)).toMatchObject(plainMeasure(ms));
+ }
+ });
+
+ test('at most one slice is set aside, and only by its margin over the next-largest excess', () => {
+ const ms = series(i => stall(9300, 12_000)(i) + stall(2500, 5_000)(i), false);
+ expect(rateMeasure(ms).excused).toEqual({ ms: 7_000, first_page: 9301, last_page: 9400 });
+ });
+
+ test('the gate message names the stall it set aside', () => {
+ const report = { ...passingReport(), engine: 'postgres' as const };
+ report.import.wall = rateMeasure(series(stall(9300, 12_000), false));
+ const gate = evaluateScaleGates(report, ENFORCE).results.find(r => r.gate === 'import_rate')!;
+ expect(gate.status).toBe('pass');
+ expect(gate.message).toContain('Both ratios leave out one 12000 ms stall in pages 9301-9400 that the rest of the import does not repeat (#7939).');
+ });
+ });
+
test('a failed data check (no-op re-import wrote rows) exits 1 naming the check', () => {
const report = passingReport();
report.data = [{ check: 'noop_reimport_writes_nothing', status: 'fail', detail: 're-importing the unchanged corpus imported 3 page(s)' }];