Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

ci: review fixes: pre-provision exits report left=0, unanswered tool calls never count, #7939 gate change withdrawn

garrytand3fe677b050f
diff --git a/scripts/ci-ubicloud.ts b/scripts/ci-ubicloud.ts
index ada0e0f924..2760a4d060 100755
--- a/scripts/ci-ubicloud.ts
+++ b/scripts/ci-ubicloud.ts
@@ -65,7 +65,10 @@
* (left: this run's VMs still listed; others: this owner's other VMs, for
* information), also appended to UBI_LEFT_FILE when set
* (scripts/pregate-remote-gate.sh reads it). A left other than 0 fails a run
- * that would otherwise pass; --keep reports its VMs as left=<n> kept=1.
+ * that would otherwise pass; --keep reports its VMs as left=<n> kept=1. A run
+ * that exits before it requests any VM (a missing key or owner, a broken
+ * inventory) prints left=0 vms=0, so pregate reads its failure as the gate's,
+ * and a missing line still means a crash after VMs were requested.
*/
import { spawnSync, type ChildProcess } from "node:child_process";
@@ -207,6 +210,8 @@ function sh(cmd: string, args: string[]): string {
const lines = (text: string) => text.split(/\s+/).map((l) => l.trim()).filter(Boolean);
const children = new Map<ChildProcess, string>();
+let vmRequested = false;
+let leftReported = false;
/** Run the runner script; output goes to `out` (a path) or is returned (scripts/ubicloud/run-script.ts). */
function runner(args: string[], opts: Omit<RunScriptOpts, "cwd" | "onSpawn" | "onClose"> = {}): Promise<{ code: number; stdout: string }> {
@@ -495,6 +500,7 @@ async function main() {
const setupLog = join(runDir, "logs", `setup-${vm.name}.log`);
if (tornDown) return;
vm.requested = true;
+ vmRequested = true;
appendFileSync(ledger, `${vm.name} ${opts.location}\n`);
const up = await runner(["up", "-n", vm.name, "-s", opts.size, "-l", opts.location, ...(opts.image === "stock" ? [] : ["-b", opts.image])], { out: setupLog });
if (tornDown) return;
@@ -679,6 +685,7 @@ async function leftover(owner: string, names: string[], kept: boolean): Promise<
const line = `UBICLOUD: left=${left} owner=${owner} vms=${names.length} ${kept ? "kept=1" : `others=${others}`}`;
console.log(line);
if (process.env.UBI_LEFT_FILE) appendFileSync(process.env.UBI_LEFT_FILE, line + "\n");
+ leftReported = true;
return left === 0;
}
@@ -702,6 +709,12 @@ function sortedObject(obj: Record<string, number>): Record<string, number> {
}
if (import.meta.main) {
+ process.on("exit", () => {
+ if (leftReported || vmRequested) return;
+ const line = `UBICLOUD: left=0 owner=${process.env.UBI_OWNER ?? ""} vms=0 others=unknown`;
+ console.log(line);
+ if (process.env.UBI_LEFT_FILE) appendFileSync(process.env.UBI_LEFT_FILE, line + "\n");
+ });
main().catch((error) => {
console.error(error);
process.exit(1);
diff --git a/scripts/mcp-tap-calls.ts b/scripts/mcp-tap-calls.ts
index 8fdec204ec..dddc645a61 100644
--- a/scripts/mcp-tap-calls.ts
+++ b/scripts/mcp-tap-calls.ts
@@ -23,9 +23,16 @@ export function tapToolCalls(log: string): string[] {
/**
* Every tool call with whether gbrain answered it with a result (#8011): a JSON-RPC `error`, a result with
* `isError: true` or no answer is not ok. The server restarts its ids with each process, so a call is ok only
- * when every response that carries its id is a successful result: a colliding error never reads as success.
+ * when every request carrying its id got an answer and every answer is a successful result: neither a
+ * colliding error nor another request's answer reads as success.
*/
export function tapToolResults(requests: string, responses: string): { name: string; ok: boolean }[] {
+ const asked = new Map<string, number>();
+ for (const m of messages(requests)) {
+ if (m.id === undefined || m.id === null || m.method === undefined) continue;
+ const key = JSON.stringify(m.id);
+ asked.set(key, (asked.get(key) ?? 0) + 1);
+ }
const answers = new Map<string, Message[]>();
for (const m of messages(responses)) {
if (m.id === undefined || m.id === null || m.method !== undefined) continue;
@@ -34,7 +41,7 @@ export function tapToolResults(requests: string, responses: string): { name: str
}
return messages(requests).filter(m => m.method === 'tools/call' && typeof m.params?.name === 'string').map(m => {
const got = answers.get(JSON.stringify(m.id)) ?? [];
- const ok = got.length > 0 && got.every(r => r.error === undefined && r.result !== undefined && r.result?.isError !== true);
+ const ok = got.length > 0 && got.length >= asked.get(JSON.stringify(m.id))! && got.every(r => r.error === undefined && r.result !== undefined && r.result?.isError !== true);
return { name: m.params!.name as string, ok };
});
}
diff --git a/scripts/scale/README.md b/scripts/scale/README.md
index 8e1493ba43..ed26c48222 100644
--- a/scripts/scale/README.md
+++ b/scripts/scale/README.md
@@ -43,13 +43,8 @@ its EXPLAIN; the JSON report and a `.explain.txt` land next to `--out`);
2 on a usage error; 3 when the harness itself crashed (not a verdict).
The import-rate gate compares the last 10% of pages' per-page cost with the
first 10% (<= 1.5x) and the total import with the time at the halfway mark
-(<= 2.5x). A tenth's per-page cost is the lowest mean over ten equal slices of
-it, and total/half sums those costs over the tenths: a stall shorter than a
-slice (a WAL checkpoint, a burst from another tenant of a CI host) once failed
-three unrelated pull requests on its own (#7939), while a cost that grows with
-the brain raises every slice. `total_ms` and `ms_at_half` stay plain sums.
-The report and its JSON `import` section carry both bases for every engine:
-wall time (`elapsed_ms`) and the import process's main-thread CPU time
+(<= 2.5x). The report and its JSON `import` section carry both bases for every
+engine: wall time (`elapsed_ms`) and the import process's main-thread CPU time
(`cpu_ms`, user + system, from each progress tick; in-process timing under
`--import-mode content`). PGLite is judged on CPU time: the database runs
synchronously on the import's main thread, so that CPU time holds all of its
diff --git a/scripts/scale/gates.ts b/scripts/scale/gates.ts
index dc3783df58..1c505515dc 100644
--- a/scripts/scale/gates.ts
+++ b/scripts/scale/gates.ts
@@ -63,30 +63,16 @@ export function importProgressPerFile(stderr: string): { wallMs: number[]; cpuMs
return { wallMs: perFile(wall), cpuMs: perFile(cpu) };
}
-/**
- * #7939: the per-page cost of each tenth of the import is the lowest mean over ten equal slices of it, and both ratios
- * use that cost (`total_vs_half` sums it over the tenths). A stall shorter than a slice of the window (a WAL checkpoint,
- * a burst from another tenant of a shared CI host) used to carry a tenth's mean past the gate on its own (1.74x, 2.1x,
- * and 2.57x total/half on three unrelated PRs); a per-page cost that grows with the brain raises every slice.
- * `total_ms` and `ms_at_half` stay the plain sums.
- */
export function rateMeasure(perPageMs: number[]): RateMeasure {
const sum = (xs: number[]) => xs.reduce((a, b) => a + b, 0);
- const sliceCost = (xs: number[]) => {
- const size = Math.max(1, Math.floor(xs.length / 10));
- let lowest = Infinity;
- for (let at = 0; at + size <= xs.length; at += size) lowest = Math.min(lowest, sum(xs.slice(at, at + size)) / size);
- return lowest;
- };
- const tenths = Array.from({ length: 10 }, (_, d) => perPageMs.slice(Math.floor(perPageMs.length * d / 10), Math.floor(perPageMs.length * (d + 1) / 10)))
- .filter(xs => xs.length > 0).map(xs => ({ pages: xs.length, cost: sliceCost(xs) }));
- const first = tenths[0]!.cost;
- const last = tenths.at(-1)!.cost;
- const work = (ts: typeof tenths) => sum(ts.map(t => t.cost * t.pages));
+ const tenth = Math.max(1, Math.floor(perPageMs.length / 10));
+ const first = sum(perPageMs.slice(0, tenth)) / tenth;
+ const last = sum(perPageMs.slice(-tenth)) / tenth;
+ const half = sum(perPageMs.slice(0, Math.floor(perPageMs.length / 2)));
+ const total = sum(perPageMs);
const round = (n: number, places: number) => Math.round(n * 10 ** places) / 10 ** places;
return { per_page_ms_first10: round(first, 1), per_page_ms_last10: round(last, 1), rate_ratio: round(last / first, 2),
- total_ms: Math.round(sum(perPageMs)), ms_at_half: Math.round(sum(perPageMs.slice(0, Math.floor(perPageMs.length / 2)))),
- total_vs_half: round(work(tenths) / work(tenths.slice(0, Math.ceil(tenths.length / 2))), 2) };
+ total_ms: Math.round(total), ms_at_half: Math.round(half), total_vs_half: round(total / half, 2) };
}
/**
diff --git a/test/scripts/pregate-remote-gate.test.ts b/test/scripts/pregate-remote-gate.test.ts
index 2690253051..69867f4726 100644
--- a/test/scripts/pregate-remote-gate.test.ts
+++ b/test/scripts/pregate-remote-gate.test.ts
@@ -143,6 +143,16 @@ describe('pregate-remote-gate.sh', () => {
expect(failing.out.trim().split('\n').at(-1)).toBe(`PREGATE_REMOTE: tree=${contentTree()} tests=5 failed=1`);
});
+ it('a ci-ubicloud run that exits before it requests any VM reports left=0 vms=0, so its failure stays a gate result, not PREGATE_REMOTE_INFRA', () => {
+ const ci = resolve(import.meta.dir, '..', '..', 'scripts/ci-ubicloud.ts');
+ const r = run(`bun ${ci}`, { UBICLOUD_API_KEY: undefined, UBICLOUD_API_TOKEN: undefined }, true);
+ expect(r.out).toContain('UBICLOUD_API_KEY (or UBICLOUD_API_TOKEN) is not set');
+ expect(r.out).toContain('UBICLOUD: left=0 owner=gbra0test vms=0 others=unknown');
+ expect(r.err).not.toContain('PREGATE_REMOTE_INFRA');
+ expect(r.err).toContain('no ci-ubicloud summary');
+ expect(r.code).toBe(2);
+ });
+
it('refuses before the gate without UBI_OWNER or when PREGATE_TREE is another tree', () => {
const marker = join(repo, '.git', 'ran');
const noOwner = run(`touch ${marker}`, { UBI_OWNER: '' });
diff --git a/test/scripts/scale-gates.test.ts b/test/scripts/scale-gates.test.ts
index 1fda7477aa..cb36e7afdb 100644
--- a/test/scripts/scale-gates.test.ts
+++ b/test/scripts/scale-gates.test.ts
@@ -161,34 +161,6 @@ describe('scale gates under --enforce', () => {
expect(importRateGate('postgres', growing)).toMatchObject({ status: 'fail', enforced: true });
});
- // #7939: one stall (a WAL checkpoint, a burst from another tenant) inside the last tenth carried its mean past the
- // gate on three unrelated PRs. Each tenth is judged by its cheapest tenth-of-a-tenth, which a stall that short
- // cannot reach and a cost that grows with the brain raises everywhere.
- const stall = (from: number, pages: number, ms: number) => (i: number) => (i >= from && i < from + pages ? ms / pages : 0);
-
- test('an 8 s stall in the last tenth passes on Postgres wall time; the same 8 s spread over every page of it fails', () => {
- const stalled = stall(PAGES * 0.95, 50, 8000);
- const gate = importRateGate('postgres', progressStream(i => ({ wall: 5 + stalled(i), cpu: 5 })));
- expect(gate.status).toBe('pass');
- expect(gate.message).toContain('import rate (wall time): last 10% per-page cost 5 ms is 1x the first 10%');
- expect(importRateGate('postgres', progressStream(i => ({ wall: lastTenth(i) ? 13 : 5, cpu: 5 })))).toMatchObject({ status: 'fail', enforced: true });
- });
-
- test('a stall in the second half passes total/half; a cost that rises over the second half fails it', () => {
- const stalled = stall(PAGES * 0.7, 80, 30_000);
- expect(importRateGate('postgres', progressStream(i => ({ wall: 5 + stalled(i), cpu: 5 }))).status).toBe('pass');
- const report = { ...passingReport(), engine: 'postgres' as const };
- report.import.wall = rateMeasure(Array.from({ length: PAGES }, (_, i) => (i >= PAGES / 2 && !lastTenth(i) ? 12 : 5)));
- expect(report.import.wall).toMatchObject({ rate_ratio: 1, total_vs_half: 3.12 });
- expect(evaluateScaleGates(report, ENFORCE).failures.map(f => f.gate)).toEqual(['import_rate']);
- });
-
- test('a slow page in every slice still counts: the cost of a tenth is a slice mean, not its fastest page', () => {
- const everyTenthSlow = progressStream(i => ({ wall: lastTenth(i) && i % 10 === 0 ? 85 : 5, cpu: 5 }));
- expect(importRateGate('postgres', everyTenthSlow)).toMatchObject({ status: 'fail' });
- expect(importRateGate('postgres', everyTenthSlow).message).toContain('last 10% per-page cost 13 ms is 2.6x');
- });
-
test('a tick without cpu_ms, or a missing tick, leaves NaN for the harness to reject', () => {
const stream = progressStream(() => ({ wall: 5, cpu: 5 })).split('\n');
stream[5] = stream[5]!.replace(/,"cpu_ms":\d+/, '');
diff --git a/test/scripts/tier2-openclaw-mcp.test.ts b/test/scripts/tier2-openclaw-mcp.test.ts
index 221385e3aa..52af565386 100644
--- a/test/scripts/tier2-openclaw-mcp.test.ts
+++ b/test/scripts/tier2-openclaw-mcp.test.ts
@@ -150,6 +150,10 @@ done
expect(tapToolResults(call(2, 'search'), ok(2))).toEqual([{ name: 'search', ok: true }]);
expect(tapToolResults([call(2, 'search'), call(2, 'query')].join('\n'), [bad(2), ok(2)].join('\n'))).toEqual([{ name: 'search', ok: false }, { name: 'query', ok: false }]);
expect(tapToolResults(`[${call(5, 'put_page')}]`, `[${ok(5)}]`)).toEqual([{ name: 'put_page', ok: true }]);
+ const ping = (id: number) => JSON.stringify({ jsonrpc: '2.0', id, method: 'ping' });
+ const pong = (id: number) => JSON.stringify({ jsonrpc: '2.0', id, result: {} });
+ expect(tapToolResults([call(2, 'search'), ping(2)].join('\n'), pong(2))).toEqual([{ name: 'search', ok: false }]);
+ expect(tapToolResults([call(2, 'search'), ping(2)].join('\n'), [ok(2), pong(2)].join('\n'))).toEqual([{ name: 'search', ok: true }]);
});
test('tapToolCalls counts only tools/call requests, batches included', () => {