Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
15.7 KB
Raw Blame
#!/usr/bin/env bun
/**
 * test/postgres-unit-arms.txt: the unit-lane suites whose PostgreSQL arm runs
 * only in persistence-validation's `unit-postgres-arms` job and the race hunt.
 *
 *   bun scripts/postgres-unit-arms.ts list                     every listed path
 *   bun scripts/postgres-unit-arms.ts shard <n> <m> [--postgres-only]
 *                                                              shard n of m, balanced by measured runtime
 *   bun scripts/postgres-unit-arms.ts run <n> <m> [--postgres-only]
 *                                                              run shard n of m, one file per worker at a time
 *   bun scripts/postgres-unit-arms.ts mine [--postgres-only] <job.log>…
 *                                                              rewrite the cell's weights from unit-postgres-arms job logs
 *
 * `run` starts one worker per URL in GBRAIN_ARM_DATABASE_URLS (space-separated,
 * one PostgreSQL server each; the job runs ARM_WORKERS). Workers take files
 * heaviest first from one queue, one Bun process per file with DATABASE_URL set
 * to the worker's server; each file's output is buffered and printed in queue
 * order inside a `::group::` block that ends with an `ARM_FILE` line carrying
 * its measured duration. With GBRAIN_TEST_RECEIPT_DIR set, each file writes its
 * own executed-test receipt (scripts/lib/test-env.sh describes the format).
 * `--postgres-only` sets GBRAIN_TEST_BACKEND=postgres, so
 * testBackends() suites run only their PostgreSQL side; their PGLite side runs
 * in the unit and serial lanes, and files that read DATABASE_URL directly keep
 * both arms.
 *
 * Weights are milliseconds per file mined from the `ARM_FILE` lines, so they
 * time each file beside another one, as the job runs it (a file ran 1.2-1.5x
 * its solo time on a 4-vCPU runner); older logs time each `::group::<file>`
 * timestamp to the next group or endgroup. The both-arms run (CI) reads
 * ARM_WEIGHTS, a --postgres-only run ARM_WEIGHTS_POSTGRES_ONLY. A file
 * without a weight gets the p75 (scripts/sharding.ts).
 * test/scripts/postgres-unit-arms.test.ts fails when a shard's two-worker
 * makespan passes ARM_SHARD_BUDGET_MS, and ci:ubicloud's pg-arms lane fails
 * when a shard's measured makespan passes the step timeout
 * (scripts/ubicloud/README.md).
 * Docs: docs/TESTING.md#postgres-arm-lanes.
 */
import { spawn } from 'node:child_process';
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
import { basename, join } from 'node:path';
import { loadWeights, partition, type WeightMap } from './sharding.ts';

export const ARMS_LIST = 'test/postgres-unit-arms.txt';
export const ARM_WEIGHTS = 'scripts/postgres-arm-weights.json';
export const ARM_WEIGHTS_POSTGRES_ONLY = 'scripts/postgres-arm-weights-postgres-only.json';
/** persistence-validation's unit-postgres-arms matrix size; test/scripts/postgres-unit-arms.test.ts pins the workflow to it. */
export const ARM_SHARDS = 24;
/** Files run at once per job, each against its own PostgreSQL service; the workflow lists one URL per worker. */
export const ARM_WORKERS = 2;
/** The workflow step's `timeout-minutes: 20`: a shard whose files run longer is cut off mid-list. */
export const ARM_STEP_TIMEOUT_MS = 20 * 60 * 1000;
/**
 * A shard's two-worker makespan may be at most this: on a runner twice as slow as the mined weights (the
 * dropped Bun 1.4.0 cell measured 1.3-2x) it still ends a minute inside the step timeout.
 */
export const ARM_SHARD_BUDGET_MS = (ARM_STEP_TIMEOUT_MS - 60_000) / 2;
/** Each file's command; ci:ubicloud's pg-arms lane (scripts/ubicloud/ci-item.sh) runs the same one. */
export const ARM_TEST_COMMAND = ['bun', '--no-env-file', 'test', '--timeout=120000'];
const DOCS = 'docs/TESTING.md#postgres-arm-lanes';
const PATH_RE = /^test\/[\w./-]+\.test\.ts$/;

export interface ArmsList { files: string[]; errors: string[] }

/** Parse the list; every malformed, duplicate or unsorted row is an error naming its line. */
export function parseArmsList(text: string): ArmsList {
  const files: string[] = [];
  const errors: string[] = [];
  text.split('\n').forEach((raw, i) => {
    const line = raw.trim();
    if (!line || line.startsWith('#')) return;
    if (!PATH_RE.test(line)) errors.push(`${ARMS_LIST}:${i + 1}: "${line}" is not a repo-relative test/…/*.test.ts path`);
    else if (files.includes(line)) errors.push(`${ARMS_LIST}:${i + 1}: ${line} is listed twice`);
    else if (files.length && files[files.length - 1]! > line) errors.push(`${ARMS_LIST}:${i + 1}: ${line} is out of sorted order (insert it after the last path that sorts before it)`);
    files.push(line);
  });
  return { files: [...new Set(files)], errors };
}

/** A missing list is empty: the lane guard then reports every gated arm as unlaned. */
export function readArmsList(root: string): ArmsList {
  const path = join(root, ARMS_LIST);
  if (!existsSync(path)) return { files: [], errors: [] };
  return parseArmsList(readFileSync(path, 'utf8'));
}

/**
 * Per-file durations from GitHub job logs, the median across logs. A log with
 * `ARM_FILE file=… ms=…` lines is read from those alone: its groups print when a
 * file's buffered output is flushed, so their timestamps time nothing. An older
 * log times each `##[group]<file>` line to the next group or endgroup. The
 * median balances shards on a typical run; one slow runner's outlier would
 * otherwise set a file's weight (a max did, and pushed most of the slow suites
 * into one shard).
 */
export function mineArmWeights(logs: string[]): Map<string, number> {
  const samples = new Map<string, number[]>();
  const add = (file: string, ms: number) => samples.set(file, [...(samples.get(file) ?? []), ms]);
  for (const log of logs) {
    const explicit = [...log.matchAll(/ARM_FILE file=(test\/[\w./-]+\.test\.ts) worker=\d+ ms=(\d+) rc=\d+/g)];
    if (explicit.length) {
      for (const m of explicit) add(m[1]!, Number(m[2]));
      continue;
    }
    let open: { file: string; at: number } | undefined;
    for (const line of log.split('\n')) {
      const m = /^(\d{4}-\d{2}-\d{2}T[\d:.]+Z)\s+(?:##\[group\]|::group::)(test\/[\w./-]+\.test\.ts)\s*$/.exec(line);
      const end = /^(\d{4}-\d{2}-\d{2}T[\d:.]+Z)\s+##\[endgroup\]/.exec(line);
      const at = m ? Date.parse(m[1]!) : end ? Date.parse(end[1]!) : NaN;
      if (!Number.isFinite(at) || (!m && !open)) continue;
      if (open && (m || end)) {
        add(open.file, at - open.at);
        open = undefined;
      }
      if (m) open = { file: m[2]!, at };
    }
  }
  return new Map([...samples].map(([file, ms]) => {
    const sorted = [...ms].sort((a, b) => a - b);
    const mid = Math.floor(sorted.length / 2);
    return [file, sorted.length % 2 ? sorted[mid]! : Math.round((sorted[mid - 1]! + sorted[mid]!) / 2)];
  }));
}

/** The cell's weights: the PostgreSQL-only cell falls back to the both-arms weights until its own are mined. */
export function armWeights(root: string, postgresOnly: boolean): WeightMap {
  const own = join(root, ARM_WEIGHTS_POSTGRES_ONLY);
  return loadWeights(postgresOnly && existsSync(own) ? own : join(root, ARM_WEIGHTS));
}

/** Shard n of m (1-based), as persistence-validation's unit-postgres-arms matrix runs it and ci:ubicloud's pg-arms lane reports it. */
export function armShards(root: string, files: string[], m: number, postgresOnly = false): string[][] {
  return partition(files, armWeights(root, postgresOnly), m).map(shard => [...shard].sort());
}

/** A shard's files in the order the workers take them: heaviest first (LPT), ties by path. */
export function armQueue(files: string[], weight: (file: string) => number): string[] {
  return [...files].sort((a, b) => weight(b) - weight(a) || (a < b ? -1 : 1));
}

/** When the last of `workers` workers finishes `files`, each taking the next queued file when it frees up. */
export function armMakespan(files: string[], weight: (file: string) => number, workers = ARM_WORKERS): number {
  const busy = new Array<number>(workers).fill(0);
  for (const file of armQueue(files, weight)) busy[busy.indexOf(Math.min(...busy))] += weight(file);
  return Math.max(...busy);
}

interface FileRun { file: string; worker: number; ms: number; rc: number; out: string }

/** One executed-test receipt per file, as scripts/lib/test-env.sh receipt_begin writes it. */
function beginReceipt(dir: string, lane: string, shard: number, of: number, file: string): string {
  const base = `${lane}--s${shard}-${basename(file, '.ts')}--primary`;
  let id = base;
  for (let n = 2; existsSync(join(dir, `${id}.receipt`)); n++) id = `${base}-${n}`;
  const sha = process.env.GITHUB_SHA || (Bun.spawnSync(['git', 'rev-parse', 'HEAD']).stdout.toString().trim() || 'unknown');
  const meta = {
    version: 1, lane, kind: 'primary', tag: `s${shard}-${basename(file, '.ts')}`, shard, of, arm: 'postgres', sha, root: process.cwd(),
    run_id: process.env.GITHUB_RUN_ID ?? '', run_attempt: process.env.GITHUB_RUN_ATTEMPT ?? '', job: process.env.GITHUB_JOB ?? '',
    started: Math.floor(Date.now() / 1000),
  };
  writeFileSync(join(dir, `${id}.receipt`), Object.entries(meta).map(([k, v]) => `${k}=${v}\n`).join(''));
  writeFileSync(join(dir, `${id}.files`), `${file}\n`);
  return id;
}

async function run(root: string, shard: number, of: number, postgresOnly: boolean): Promise<number> {
  const urls = (process.env.GBRAIN_ARM_DATABASE_URLS ?? '').split(/\s+/).filter(Boolean);
  if (!urls.length) {
    console.error(`::error::GBRAIN_ARM_DATABASE_URLS is empty. Why: each worker runs its files against its own PostgreSQL server. Fix: list one URL per worker (the job runs ${ARM_WORKERS}). Docs: ${DOCS}`);
    return 2;
  }
  const weights = armWeights(root, postgresOnly);
  const fallback = [...weights.values()].sort((a, b) => a - b)[Math.floor(weights.size * 0.75)] ?? 1;
  const files = armQueue(armShards(root, readArmsList(root).files, of, postgresOnly)[shard - 1] ?? [], f => weights.get(f) ?? fallback);
  if (!files.length) {
    console.error(`::error::shard ${shard}/${of} selected no files from ${ARMS_LIST}. Why: an empty shard would pass without running any PostgreSQL arm. Fix: run 'bun scripts/postgres-unit-arms.ts shard ${shard} ${of}' locally and correct the list. Docs: ${DOCS}`);
    return 1;
  }
  const receiptDir = process.env.GBRAIN_TEST_RECEIPT_DIR ?? '';
  if (receiptDir) mkdirSync(receiptDir, { recursive: true });
  const lane = process.env.GBRAIN_TEST_RECEIPT_LANE || 'pg-arms';
  const env: Record<string, string | undefined> = { ...process.env, ...(postgresOnly ? { GBRAIN_TEST_BACKEND: 'postgres' } : {}) };
  for (const name of ['GBRAIN_TEST_RECEIPT_DIR', 'GBRAIN_TEST_RECEIPT_LANE', 'GBRAIN_ARM_DATABASE_URLS']) delete env[name];
  const workers = Math.min(urls.length, files.length);
  console.log(`postgres-unit-arms: shard ${shard}/${of}, ${files.length} files, ${workers} workers${postgresOnly ? ', PostgreSQL side only (GBRAIN_TEST_BACKEND=postgres)' : ''}`);

  const results: Array<FileRun | undefined> = new Array(files.length);
  const running = new Map<number, { file: string; worker: number; chunks: Buffer[] }>();
  let next = 0;
  let printed = 0;
  const started = Date.now();
  const busy = new Array<number>(workers).fill(0);
  const flush = () => {
    for (; printed < files.length && results[printed]; printed++) {
      const r = results[printed]!;
      process.stdout.write(`::group::${r.file}\n${r.out}${r.out.endsWith('\n') || !r.out ? '' : '\n'}ARM_FILE file=${r.file} worker=${r.worker} ms=${r.ms} rc=${r.rc}\n::endgroup::\n`);
    }
  };
  const runFile = (i: number, worker: number) => new Promise<FileRun>(resolve => {
    const file = files[i]!;
    const id = receiptDir ? beginReceipt(receiptDir, lane, shard, of, file) : '';
    const args = [...ARM_TEST_COMMAND.slice(1), ...(id ? ['--reporter=junit', `--reporter-outfile=${join(receiptDir, `${id}.junit.xml`)}`] : []), file];
    const chunks: Buffer[] = [];
    running.set(i, { file, worker, chunks });
    const t0 = Date.now();
    const child = spawn(process.execPath, args, { env: { ...env, DATABASE_URL: urls[worker] }, stdio: ['ignore', 'pipe', 'pipe'] });
    child.stdout.on('data', c => chunks.push(c));
    child.stderr.on('data', c => chunks.push(c));
    const done = (rc: number, note = '') => {
      running.delete(i);
      if (id) writeFileSync(join(receiptDir, `${id}.receipt`), `exit=${rc}\n`, { flag: 'a' });
      resolve({ file, worker, ms: Date.now() - t0, rc, out: Buffer.concat(chunks).toString('utf8') + note });
    };
    child.on('error', error => done(127, `postgres-unit-arms: could not start bun: ${error.message}\n`));
    child.on('close', (code, signal) => done(code ?? 1, signal ? `postgres-unit-arms: bun ended on ${signal}\n` : ''));
  });
  for (const signal of ['SIGINT', 'SIGTERM'] as const) {
    process.on(signal, () => {
      flush();
      for (const r of running.values()) process.stdout.write(`::group::${r.file} (interrupted)\n${Buffer.concat(r.chunks).toString('utf8')}\n::endgroup::\n`);
      console.log(`::error::shard ${shard}/${of} stopped on ${signal} with ${files.length - printed} file(s) unfinished; the step timeout (${ARM_STEP_TIMEOUT_MS / 60_000} minutes) or a cancellation ended it.`);
      process.exit(130);
    });
  }
  await Promise.all(Array.from({ length: workers }, async (_, worker) => {
    while (next < files.length) {
      const i = next++;
      const r = await runFile(i, worker);
      busy[worker] += r.ms;
      results[i] = r;
      flush();
    }
  }));
  const failed = results.filter(r => r!.rc !== 0).map(r => r!.file);
  const backend = postgresOnly ? 'GBRAIN_TEST_BACKEND=postgres ' : '';
  for (const file of failed) console.log(`::error file=${file}::PostgreSQL arm failed. Reproduce: DATABASE_URL=<pgvector test database> GBRAIN_TEST_ALLOW_DATABASE_URL=1 ${backend}${ARM_TEST_COMMAND.join(' ')} ${file}`);
  console.log(`ARM_SHARD shard=${shard}/${of} workers=${workers} files=${files.length} wall_ms=${Date.now() - started} busy_ms=${busy.join(',')} failed=${failed.length}`);
  return failed.length ? 1 : 0;
}

async function main(argv: string[]): Promise<number> {
  const root = join(import.meta.dir, '..');
  const postgresOnly = argv.includes('--postgres-only');
  const [command, ...rest] = argv.filter(a => a !== '--postgres-only');
  const list = readArmsList(root);
  if (command !== 'mine' && list.errors.length) {
    console.error(`${list.errors.join('\n')}\n  Why: the Postgres-arm lanes read this list; a bad row would drop or duplicate an arm.\n  Fix: correct the named line in ${ARMS_LIST}\n  Docs: ${DOCS}`);
    return 1;
  }
  if (command === 'list') {
    console.log(list.files.join('\n'));
    return 0;
  }
  if (command === 'shard' || command === 'run') {
    const [n, m] = rest.map(Number);
    if (rest.length !== 2 || !Number.isInteger(n) || !Number.isInteger(m) || n! < 1 || n! > m!) {
      console.error(`usage: bun scripts/postgres-unit-arms.ts ${command} <n> <m> [--postgres-only]   (1 <= n <= m)`);
      return 2;
    }
    if (command === 'run') return run(root, n!, m!, postgresOnly);
    console.log(armShards(root, list.files, m!, postgresOnly)[n! - 1]!.join('\n'));
    return 0;
  }
  if (command === 'mine' && rest.length) {
    const target = postgresOnly ? ARM_WEIGHTS_POSTGRES_ONLY : ARM_WEIGHTS;
    const mined = mineArmWeights(rest.map(path => readFileSync(path, 'utf8')));
    const sorted = Object.fromEntries([...mined].sort(([a], [b]) => (a < b ? -1 : 1)));
    writeFileSync(join(root, target), `${JSON.stringify(sorted, null, 2)}\n`);
    const missing = list.files.filter(f => !mined.has(f));
    console.log(`postgres-arm weights: ${mined.size} files mined into ${target}${missing.length ? `; no timing for ${missing.join(', ')}` : ''}`);
    return 0;
  }
  console.error('usage: bun scripts/postgres-unit-arms.ts list | shard <n> <m> [--postgres-only] | run <n> <m> [--postgres-only] | mine [--postgres-only] <job.log>…');
  return 2;
}

if (import.meta.main) process.exit(await main(process.argv.slice(2)));