Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
scale: import_rate sets aside one stall, never recurring work (#7939)
garrytan077de53a2a88
diff --git a/scripts/scale/gates.ts b/scripts/scale/gates.tsindex 1c505515dc..85767a8061 100644--- a/scripts/scale/gates.ts+++ b/scripts/scale/gates.ts@@ -34,7 +34,10 @@ export const RATE_RATIO_MAX = 1.5;export const RATE_MIN_PAGES = 1000;export const TOTAL_VS_HALF_MAX = 2.5;-/** One timing basis of the import: per-page cost of the first and last 10%, and total vs the halfway mark. */+/**+ * One timing basis of the import: per-page cost of the first and last 10%, and total vs the halfway mark.+ * The two ratios leave out the one stall `excusedStall` sets aside; `total_ms` and `ms_at_half` are the measured sums.+ */export interface RateMeasure {per_page_ms_first10: number;per_page_ms_last10: number;@@ -42,8 +45,12 @@ export interface RateMeasure {total_ms: number;ms_at_half: number;total_vs_half: number;+ excused?: ExcusedStall;}+/** The milliseconds the rate measure set aside, and the 1-based pages of the slice that held them. */+export interface ExcusedStall { ms: number; first_page: number; last_page: number }+/*** Per-file wall and CPU milliseconds from `gbrain import --progress-json --progress-interval 0`* stderr: one `import.files` tick per file, each with cumulative `elapsed_ms` and `cpu_ms`.@@ -63,16 +70,70 @@ export function importProgressPerFile(stderr: string): { wallMs: number[]; cpuMsreturn { wallMs: perFile(wall), cpuMs: perFile(cpu) };}+/**+ * #7939: one stall of the import (a paused database container, a WAL checkpoint, a burst from another tenant of a+ * shared CI host) lands in one page's time and once carried a whole tenth's mean past the gate on its own. Work that+ * grows with the brain does not look like that: it recurs. So the measure sets aside at most one stall per import,+ * and only the part of it no other part of the import repeats:+ *+ * - Each tenth of the import is cut into ten slices. A slice's excess is its mean per-page cost above the median+ * slice of its tenth. The slice with the largest excess is lowered to the second-largest excess anywhere in the+ * import, so a cost that shows up twice (two rebuilds, two stalls) is set aside only by the margin between them.+ * - The lowered tenth never drops below the Theil-Sen line through the ten tenth means, so a cost that lands once in+ * every tenth and grows with the brain is kept in full: those tenth means rise on that line.+ *+ * Nothing else changes: every other slice counts, and with no stall the measure is the plain mean of each tenth.+ */+export function excusedStall(perPageMs: number[]): ExcusedStall | undefined {+ const n = perPageMs.length;+ if (n < 100) return undefined;+ const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);+ const median = (xs: number[]) => {+ const s = [...xs].sort((a, b) => a - b);+ const mid = s.length >> 1;+ return s.length % 2 ? s[mid]! : (s[mid - 1]! + s[mid]!) / 2;+ };+ const tenParts = (from: number, to: number) => Array.from({ length: 10 }, (_, k) => {+ const lo = from + Math.floor((to - from) * k / 10);+ const hi = from + Math.floor((to - from) * (k + 1) / 10);+ return { lo, hi, mean: sum(perPageMs.slice(lo, hi)) / (hi - lo) };+ });+ const tenths = tenParts(0, n);+ const slices = tenths.flatMap((tenth, d) => {+ const parts = tenParts(tenth.lo, tenth.hi);+ const typical = median(parts.map(s => s.mean));+ return parts.map(s => ({ ...s, d, excess: s.mean - typical }));+ }).sort((a, b) => b.excess - a.excess);+ const [worst, next] = [slices[0]!, slices[1]!];+ const means = tenths.map(t => t.mean);+ const slopes = means.flatMap((a, i) => means.slice(i + 1).map((b, k) => (b - a) / (k + 1)));+ const slope = median(slopes);+ const trend = median(means.map((m, d) => m - slope * d)) + slope * worst.d;+ const tenth = tenths[worst.d]!;+ const pages = tenth.hi - tenth.lo;+ const lowered = tenth.mean - (worst.excess - next.excess) * (worst.hi - worst.lo) / pages;+ const ms = Math.round((tenth.mean - Math.min(tenth.mean, Math.max(lowered, trend))) * pages);+ return ms > 0 ? { ms, first_page: worst.lo + 1, last_page: worst.hi } : undefined;+}+export function rateMeasure(perPageMs: number[]): RateMeasure {const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);- const tenth = Math.max(1, Math.floor(perPageMs.length / 10));- const first = sum(perPageMs.slice(0, tenth)) / tenth;- const last = sum(perPageMs.slice(-tenth)) / tenth;- const half = sum(perPageMs.slice(0, Math.floor(perPageMs.length / 2)));- const total = sum(perPageMs);+ const excused = excusedStall(perPageMs);+ const judged = [...perPageMs];+ if (excused) {+ const stalled = judged.slice(excused.first_page - 1, excused.last_page);+ const keep = 1 - excused.ms / sum(stalled);+ stalled.forEach((ms, k) => { judged[excused.first_page - 1 + k] = ms * keep; });+ }+ const tenth = Math.max(1, Math.floor(judged.length / 10));+ const first = sum(judged.slice(0, tenth)) / tenth;+ const last = sum(judged.slice(-tenth)) / tenth;+ const halfPages = Math.floor(perPageMs.length / 2);const round = (n: number, places: number) => Math.round(n * 10 ** places) / 10 ** places;return { per_page_ms_first10: round(first, 1), per_page_ms_last10: round(last, 1), rate_ratio: round(last / first, 2),- total_ms: Math.round(total), ms_at_half: Math.round(half), total_vs_half: round(total / half, 2) };+ total_ms: Math.round(sum(perPageMs)), ms_at_half: Math.round(sum(perPageMs.slice(0, halfPages))),+ total_vs_half: round(sum(judged) / sum(judged.slice(0, halfPages)), 2),+ ...(excused ? { excused } : {}) };}/**@@ -174,10 +235,11 @@ export function evaluateScaleGates(report: ScaleReport, policy: GatePolicy): Gatconst basis = importRateBasis(report.engine);const other = basis === 'cpu' ? 'wall' : 'cpu';- const { rate_ratio, total_vs_half, per_page_ms_first10, per_page_ms_last10 } = report.import[basis];+ const { rate_ratio, total_vs_half, per_page_ms_first10, per_page_ms_last10, excused } = report.import[basis];add('import_rate', rate_ratio <= RATE_RATIO_MAX && total_vs_half <= TOTAL_VS_HALF_MAX, report.pages >= RATE_MIN_PAGES,`import rate (${basis} time): last 10% per-page cost ${per_page_ms_last10} ms is ${rate_ratio}x the first 10% (${per_page_ms_first10} ms; gate <= ${RATE_RATIO_MAX}); `+ `total import time is ${total_vs_half}x the time at the halfway mark (gate <= ${TOTAL_VS_HALF_MAX}). `+ + (excused ? `Both ratios leave out one ${excused.ms} ms stall in pages ${excused.first_page}-${excused.last_page} that the rest of the import does not repeat (#7939). ` : '')+ `Not judged on ${report.engine}: ${other} time ratio ${report.import[other].rate_ratio}, total/half ${report.import[other].total_vs_half}. `+ `A rising per-page cost means import slows as the brain grows, usually stale planner statistics during import. Reproduce: ${repro}`);diff --git a/test/scripts/scale-gates.test.ts b/test/scripts/scale-gates.test.tsindex 52097a4d7b..1f262578e2 100644--- a/test/scripts/scale-gates.test.ts+++ b/test/scripts/scale-gates.test.ts@@ -8,7 +8,7 @@import { describe, expect, test } from 'bun:test';import {CEILING_PAGES, CEILINGS_MS, ceilingsFor, evaluateScaleGates, importProgressPerFile, RATE_MIN_PAGES, HEADLINE_OP, HOT_TABLES, KEY_PLAN_OPS, phaseLimitsMs, PLANNER_HEALTH_ENFORCED, PLANNER_STATS_MIN_ROWS,- rateMeasure, resultHits, verdictLines, type GatePolicy, type RateMeasure, type ScaleReport,+ RATE_RATIO_MAX, rateMeasure, resultHits, TOTAL_VS_HALF_MAX, verdictLines, type GatePolicy, type RateMeasure, type ScaleReport,} from '../../scripts/scale/gates.ts';const ENFORCE: GatePolicy = { enforce: true, enforcePlanner: PLANNER_HEALTH_ENFORCED, enforceCeilings: false };@@ -172,6 +172,99 @@ describe('scale gates under --enforce', () => {});});+ // #7939: one stall (a paused database container, a WAL checkpoint) inside the last tenth carried its mean past the+ // gate on unrelated PRs. The measure sets aside at most one stall, and only the part of it nothing else in the+ // import repeats; work that recurs, which is how an import goes quadratic, keeps the plain measure's sensitivity.+ describe('import rate sets aside one stall and never recurring work (#7939)', () => {+ const PAGES = 10_000;+ let state = 7939;+ const jitter = Array.from({ length: PAGES }, () => {+ state = (state + 0x6D2B79F5) | 0;+ let x = Math.imul(state ^ (state >>> 15), 1 | state);+ x = (x + Math.imul(x ^ (x >>> 7), 61 | x)) ^ x;+ return 0.8 + 0.4 * (((x ^ (x >>> 14)) >>> 0) / 4294967296);+ });+ const series = (extra: (i: number) => number, noisy = true) => Array.from({ length: PAGES }, (_, i) => 5 * (noisy ? jitter[i]! : 1) + extra(i));+ /** Master's measure before #7939: plain means of the first and last tenth, plain total over the halfway sum. */+ const plainMeasure = (ms: number[]) => {+ const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);+ const round = (n: number) => Math.round(n * 100) / 100;+ return { rate_ratio: round(sum(ms.slice(-PAGES / 10)) / sum(ms.slice(0, PAGES / 10))), total_vs_half: round(sum(ms) / sum(ms.slice(0, PAGES / 2))) };+ };+ const verdict = (m: { rate_ratio: number; total_vs_half: number }) => (m.rate_ratio <= RATE_RATIO_MAX && m.total_vs_half <= TOTAL_VS_HALF_MAX ? 'pass' : 'fail');+ const stall = (page: number, ms: number) => (i: number) => (i === page ? ms : 0);+ const rebuild = (every: number, msPerPageSoFar: number, phase = 0) => (i: number) => (i > 0 && (i - phase) % every === 0 ? msPerPageSoFar * i : 0);+ const alternatingGrowth = (msPerPageSoFar: number) => (i: number) => (Math.floor(i / 100) % 2 ? msPerPageSoFar * i : 0);+ const linear = (msPerPageSoFar: number) => (i: number) => msPerPageSoFar * i;++ const CASES: Array<[string, 'pass' | 'fail', 'pass' | 'fail', (i: number) => number]> = [+ ['no regression', 'pass', 'pass', () => 0],+ ['one 12 s stall at page 9,300', 'fail', 'pass', stall(9300, 12_000)],+ ['one 30 s stall in the last slice', 'fail', 'pass', stall(9950, 30_000)],+ ['one 30 s stall in the second half', 'fail', 'pass', stall(7000, 30_000)],+ ['a rebuild every 200 pages costing 0.5 ms x pages so far', 'fail', 'fail', rebuild(200, 0.5)],+ ['the same rebuild every 150 pages', 'fail', 'fail', rebuild(150, 0.5)],+ ['growth only on alternating 100-page blocks', 'fail', 'fail', alternatingGrowth(0.001)],+ ['a plain O(n) per-page cost', 'fail', 'fail', linear(0.0005)],+ ['a rebuild once per tenth (every 1,000 pages), one slice per tenth', 'fail', 'fail', rebuild(1000, 0.5)],+ ['the same rebuild half a tenth later', 'fail', 'fail', rebuild(1000, 0.5, 500)],+ ['two 12 s stalls in the last tenth', 'fail', 'fail', i => stall(9300, 12_000)(i) + stall(9700, 12_000)(i)],+ ['every page of the last tenth 60% slower', 'fail', 'fail', i => (i >= 9000 ? 3 : 0)],+ ];+ test.each(CASES)('%s: before #7939 %s, now %s', (_name, before, now, extra) => {+ const ms = series(extra);+ expect(verdict(plainMeasure(ms))).toBe(before);+ expect(verdict(rateMeasure(ms))).toBe(now);+ });++ // Equal sensitivity: the smallest amplitude that fails the gate, found by bisection, is the plain measure's to within+ // 1% (the page-to-page jitter of the slice that is lowered; 0.8% at most here).+ const failsAt = (measure: (ms: number[]) => { rate_ratio: number; total_vs_half: number }, family: (k: number) => (i: number) => number) => {+ let lo = 0;+ let hi = 1;+ while (verdict(measure(series(family(hi)))) === 'pass') hi *= 2;+ for (let step = 0; step < 30; step++) {+ const mid = (lo + hi) / 2;+ if (verdict(measure(series(family(mid)))) === 'fail') hi = mid; else lo = mid;+ }+ return hi;+ };+ const RECURRING: Array<[string, (k: number) => (i: number) => number]> = [+ ['a rebuild every 200 pages', k => rebuild(200, k)],+ ['a rebuild every 150 pages', k => rebuild(150, k)],+ ['growth only on alternating 100-page blocks', k => alternatingGrowth(k)],+ ['a plain O(n) per-page cost', k => linear(k)],+ ['a rebuild once per tenth', k => rebuild(1000, k)],+ ['a rebuild once per tenth, half a tenth later', k => rebuild(1000, k, 500)],+ ];+ test.each(RECURRING)('%s fails at the same size as under the plain measure', (_name, family) => {+ const ratio = failsAt(rateMeasure, family) / failsAt(plainMeasure, family);+ expect(ratio).toBeGreaterThanOrEqual(1);+ expect(ratio).toBeLessThan(1.01);+ });++ test('a cost that grows with the brain sets nothing aside: the measure is the plain one', () => {+ for (const extra of [linear(0.0005), rebuild(1000, 0.5), (i: number) => (i >= 9000 ? 3 : 0)]) {+ const ms = series(extra, false);+ expect(rateMeasure(ms).excused).toBeUndefined();+ expect(rateMeasure(ms)).toMatchObject(plainMeasure(ms));+ }+ });++ test('at most one slice is set aside, and only by its margin over the next-largest excess', () => {+ const ms = series(i => stall(9300, 12_000)(i) + stall(2500, 5_000)(i), false);+ expect(rateMeasure(ms).excused).toEqual({ ms: 7_000, first_page: 9301, last_page: 9400 });+ });++ test('the gate message names the stall it set aside', () => {+ const report = { ...passingReport(), engine: 'postgres' as const };+ report.import.wall = rateMeasure(series(stall(9300, 12_000), false));+ const gate = evaluateScaleGates(report, ENFORCE).results.find(r => r.gate === 'import_rate')!;+ expect(gate.status).toBe('pass');+ expect(gate.message).toContain('Both ratios leave out one 12000 ms stall in pages 9301-9400 that the rest of the import does not repeat (#7939).');+ });+ });+test('a failed data check (no-op re-import wrote rows) exits 1 naming the check', () => {const report = passingReport();report.data = [{ check: 'noop_reimport_writes_nothing', status: 'fail', detail: 're-importing the unchanged corpus imported 3 page(s)' }];