Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.7 KB
Raw Blame
#!/usr/bin/env bun
/**
 * Run one CI setup command with a per-attempt bound and one retry.
 *
 * A network-bound setup step (`bun install`, Playwright's browser and OS
 * dependency install) normally takes seconds, and on a bad registry, mirror or
 * CDN connection it can crawl until the job timeout cancels it, with nothing
 * naming the cause (.195's 805 s `bun install` on darwin-x64; .204's 12-minute
 * `playwright install --with-deps chromium` on admin-browser). This runs the
 * command in its own process group and ends an attempt (SIGTERM to the group,
 * so `sudo apt-get` under it is relayed the signal, then SIGKILL after a grace
 * period, and the attempt ends only once that whole group is gone) when it stalls: still running at `--attempt-s`, or silent on stdout
 * and stderr for `--idle-s`. A stalled attempt is retried once after a pause,
 * and a second failure stops with a named `::error::`. `--total-s` caps every
 * attempt and pause together, so a slow download that keeps printing progress
 * runs to the budget instead of being restarted from zero, and no retry starts
 * without time left for it. A failure that is not a stall is retried once too
 * and then reported with the command's exit code. A bound that fires after
 * the command already exited is not a stall: on a loaded runner this process
 * can be descheduled past the bound while its child finishes, and its exit
 * event then queues behind the expired timer, so on Linux the timer reads the
 * child's /proc state first and leaves a zombie or reaped child to that exit
 * event. No dependency: it runs before they are installed.
 *
 * Usage: bun scripts/ci-bounded-run.ts (--attempt-s <n> | --total-s <n>) [--idle-s <n>] [--retry-s <n>] [--hint <text>] -- <command> [args...]
 */
import { spawn } from 'node:child_process';
import { readdirSync, readFileSync } from 'node:fs';

export interface BoundedRun {
  argv: string[];
  /** The command as the annotations name it. */
  label: string;
  /** Wall-clock bound on one attempt. */
  attemptSeconds?: number;
  /** An attempt that prints nothing on stdout or stderr for this long is stalled. */
  idleSeconds?: number;
  /** Wall-clock budget for every attempt and pause together; no retry starts once it is spent. */
  totalSeconds?: number;
  retrySeconds: number;
  /** Why a stall is unusual for this command, appended to the stall line. */
  hint?: string;
  graceSeconds?: number;
}

const ATTEMPTS = 2;

/** Whether process group `pgid` still has a member that is not a zombie: a SIGKILLed process dies only when it next runs. */
function groupLive(pgid: number): boolean {
  if (process.platform !== 'linux') {
    try { process.kill(-pgid, 0); return true; } catch { return false; }
  }
  for (const entry of readdirSync('/proc')) {
    if (!/^\d+$/.test(entry)) continue;
    try {
      const stat = readFileSync(`/proc/${entry}/stat`, 'utf8');
      const [state, , pgrp] = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
      if (Number(pgrp) === pgid && !['Z', 'X'].includes(state!)) return true;
    } catch { /* gone between the listing and the read */ }
  }
  return false;
}

/** Waits, at most `boundMs`, until a killed attempt's whole group is gone, so no retry or exit overlaps what it left running. */
async function groupGone(pgid: number, boundMs: number): Promise<void> {
  const until = Date.now() + boundMs;
  while (groupLive(pgid) && Date.now() < until) await Bun.sleep(10);
}

type Ending = 'exit' | 'attempt' | 'idle' | 'budget';

function attempt(run: BoundedRun, boundMs: number, budgetBound: boolean): Promise<{ code: number | null; ending: Ending }> {
  const grouped = process.platform !== 'win32';
  const watched = run.idleSeconds !== undefined;
  const child = spawn(run.argv[0]!, run.argv.slice(1), { stdio: ['ignore', watched ? 'pipe' : 'inherit', watched ? 'pipe' : 'inherit'], detached: grouped });
  const signal = (name: NodeJS.Signals) => {
    try { if (grouped && child.pid) process.kill(-child.pid, name); else child.kill(name); } catch { /* the group is gone */ }
  };
  let ending: Ending = 'exit', kill: ReturnType<typeof setTimeout> | undefined, idle: ReturnType<typeof setTimeout> | undefined;
  const exited = () => {
    if (process.platform !== 'linux') return false;
    try {
      const stat = readFileSync(`/proc/${child.pid}/stat`, 'utf8');
      return ['Z', 'X'].includes(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0]!);
    } catch {
      return true;
    }
  };
  const end = (why: Ending) => {
    if (ending !== 'exit' || exited()) return;
    ending = why;
    signal('SIGTERM');
    kill = setTimeout(() => signal('SIGKILL'), (run.graceSeconds ?? 10) * 1000);
  };
  const bound = Number.isFinite(boundMs) ? setTimeout(() => end(budgetBound ? 'budget' : 'attempt'), boundMs) : undefined;
  const quiet = () => { clearTimeout(idle); idle = setTimeout(() => end('idle'), run.idleSeconds! * 1000); };
  if (watched) {
    quiet();
    child.stdout!.on('data', chunk => { process.stdout.write(chunk); quiet(); });
    child.stderr!.on('data', chunk => { process.stderr.write(chunk); quiet(); });
  }
  const settle = () => { clearTimeout(bound); clearTimeout(kill); clearTimeout(idle); };
  return new Promise(resolve => {
    child.once('error', () => { settle(); resolve({ code: 127, ending }); });
    child.once('exit', async code => {
      settle();
      if (ending !== 'exit') { signal('SIGKILL'); if (grouped && child.pid) await groupGone(child.pid, (run.graceSeconds ?? 10) * 1000); }
      resolve({ code, ending });
    });
  });
}

export async function runBounded(run: BoundedRun): Promise<number> {
  const deadline = run.totalSeconds !== undefined ? Date.now() + run.totalSeconds * 1000 : Infinity;
  const hint = run.hint ? ` (${run.hint})` : '';
  for (let n = 1; ; n++) {
    const started = Date.now();
    const attemptMs = run.attemptSeconds !== undefined ? run.attemptSeconds * 1000 : Infinity;
    const { code, ending } = await attempt(run, Math.min(attemptMs, deadline - started), deadline - started <= attemptMs);
    if (code === 0 && ending === 'exit') return 0;
    const why = {
      exit: `exited ${code} after ${Math.round((Date.now() - started) / 1000)}s`,
      attempt: `stalled: no result after ${run.attemptSeconds}s${hint}`,
      idle: `stalled: no output for ${run.idleSeconds}s${hint}`,
      budget: `was still running when its ${run.totalSeconds}s budget ran out${hint}`,
    }[ending];
    if (n >= ATTEMPTS || deadline - Date.now() <= run.retrySeconds * 1000) {
      console.error(`::error::${run.label} ${why} on attempt ${n} of ${ATTEMPTS}; giving up`);
      return 1;
    }
    console.error(`::warning::${run.label} attempt ${n} ${why}; retrying in ${run.retrySeconds}s`);
    await Bun.sleep(run.retrySeconds * 1000);
  }
}

if (import.meta.main) {
  const args = process.argv.slice(2), split = args.indexOf('--');
  const option = (name: string) => { const i = args.indexOf(name); return i >= 0 && i < split ? args[i + 1] : undefined; };
  const seconds = (name: string) => { const v = option(name); return v === undefined ? undefined : Number(v); };
  const argv = split >= 0 ? args.slice(split + 1) : [];
  const attemptSeconds = seconds('--attempt-s'), idleSeconds = seconds('--idle-s'), totalSeconds = seconds('--total-s');
  const valid = [attemptSeconds, idleSeconds, totalSeconds].every(v => v === undefined || v > 0);
  if (!argv.length || !valid || (attemptSeconds === undefined && totalSeconds === undefined)) {
    console.error('usage: bun scripts/ci-bounded-run.ts (--attempt-s <n> | --total-s <n>) [--idle-s <n>] [--retry-s <n>] [--hint <text>] -- <command> [args...]');
    process.exit(2);
  }
  process.exit(await runBounded({ argv, label: argv.join(' '), attemptSeconds, idleSeconds, totalSeconds, retrySeconds: Number(option('--retry-s') ?? 10), hint: option('--hint') }));
}