Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
20.6 KB
Raw Blame
#!/usr/bin/env bun
/** Mine timings from successful CI execution in its real execution mode.
 * Unit: consecutive headers + final Bun summary, milliseconds.
 * Serial: per-file runner PASS records, seconds (not buffered log timestamps).
 * E2E: per-file Bun summaries, milliseconds. Partial selections merge old weights;
 * explicit full-profile runs replace them after source-corpus validation.
 * Several --run values, in any order, give each file its median over those runs: one run's
 * timings carry that run's runner speeds (whole runs differ by 0.75-1.3x and
 * two-shard jobs within one run by 0.7-1.7x), so a lone fast run under-predicts
 * every shard.
 * Every mode drops entries for files git no longer tracks, so a deleted or
 * renamed test leaves the map whatever the runs measured.
 * Refresh after large test additions or sustained shard imbalance.
 * Seam: GBRAIN_MINE_ROOT (the checkout whose tracked files bound the map).
 */
import { spawnSync } from "node:child_process";
import { readFileSync, writeFileSync, renameSync, rmSync, existsSync, mkdtempSync, mkdirSync } from "node:fs";
import { basename, dirname, join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { createHash } from "node:crypto";
import { fileURLToPath } from "node:url";
import { loadWeights } from "./sharding.ts";

const ROOT = process.env.GBRAIN_MINE_ROOT ?? resolve(dirname(fileURLToPath(import.meta.url)), "..");
export type Lane = "unit" | "serial" | "e2e";
type E2EProfile = "selected" | "full";
const OUTPUTS = { unit: "test-weights", serial: "serial-weights", e2e: "e2e-weights" };
interface TimingEvent { job: string; timestampMs: number; file: string; summaryCount?: number; failures?: number }
interface LogLine { job: string; step: string; timestampMs: number; text: string }
function logLines(raw: string): LogLine[] {
  return raw.split("\n").flatMap(line => {
    const m = /^([^\t]+)\t([^\t]*)\t(\d{4}-\d{2}-\d{2}T\S+Z)\s+(.*)$/.exec(line);
    if (!m || !Number.isFinite(Date.parse(m[3]!))) return [];
    return [{ job: m[1]!.trim(), step: m[2]!.trim(), timestampMs: Date.parse(m[3]!), text: m[4]!.replace(/\x1b\[[0-9;]*m/g, "") }];
  });
}
/** The shards a test.yml unit job ran: `test (3)` alone, `test (3, 4)` packed by scripts/test-shard-pack.sh. */
function unitJobShards(job: string): string[] | undefined {
  return /^test \((\d+(?:, \d+)*)\)$/.exec(job)?.[1]!.split(", ");
}
function isJob(job: string, lane: Lane, e2eProfile: E2EProfile = "selected"): boolean {
  if (lane === "unit") return unitJobShards(job) !== undefined;
  if (lane === "serial") return /^serial-tests(?: \(\d+\))?$/.test(job);
  if (e2eProfile === "full") return /^coverage-full-e2e(?: \(\d+\))?$/.test(job);
  return /^Selected E2E \(diff-relevant\)(?: \(\d+\)| \d+)?$/.test(job);
}

/** File starts and the LAST summary per unit job; nested CLI summaries are ignored. */
export function parseLog(raw: string): TimingEvent[] {
  const events: TimingEvent[] = [];
  const ends = new Map<string, TimingEvent>();
  const failures = new Map<string, number>();
  for (const line of logLines(raw)) {
    if (!isJob(line.job, "unit")) continue;
    // Captured artifacts retain Bun's raw workflow command; GitHub's rendered
    // logs rewrite that same command to the bracketed form.
    const m = /^(?:##\[group\]|::group::)((?:test|evals)\/[^\s:]+\.test\.ts):?\s*$/.exec(line.text);
    if (m) events.push({ job: line.job, timestampMs: line.timestampMs, file: m[1]! });
    const fail = /^\s*(\d+) fail\s*$/.exec(line.text);
    if (fail) failures.set(line.job, Number(fail[1]));
    const summary = /^Ran \d+ tests? across (\d+) files?\./.exec(line.text);
    if (summary) ends.set(line.job, { job: line.job, timestampMs: line.timestampMs, file: "", summaryCount: Number(summary[1]), failures: failures.get(line.job) });
  }
  return [...events, ...ends.values()];
}
export function computeWeights(events: TimingEvent[]): Map<string, number> {
  const byJob = new Map<string, TimingEvent[]>();
  for (const e of events) {
    if (!byJob.has(e.job)) byJob.set(e.job, []);
    byJob.get(e.job)!.push(e);
  }
  const weights = new Map<string, number>();
  for (const list of byJob.values()) {
    for (let i = 0; i + 1 < list.length; i++) {
      const a = list[i]!, b = list[i + 1]!;
      if (!a.file || b.timestampMs < a.timestampMs) continue;
      weights.set(a.file, Math.max(weights.get(a.file) ?? 0, Math.round(b.timestampMs - a.timestampMs)));
    }
  }
  return weights;
}

/** Refuse partial/failed sources before any output is replaced. */
export function mineWeights(raw: string, lane: Lane, opts: { expectedJobs?: readonly string[]; e2eProfile?: E2EProfile; expectedFiles?: readonly string[]; requireCapture?: boolean } = {}): Map<string, number> {
  const full = opts.e2eProfile === "full";
  const fullPaths = new Map<string, string>();
  if (full) {
    if (lane !== "e2e" || !opts.expectedFiles?.length) throw new Error("full E2E requires an authoritative source corpus");
    for (const file of opts.expectedFiles) {
      if (fullPaths.has(basename(file))) throw new Error(`full E2E: ambiguous or duplicate basename ${basename(file)}`);
      fullPaths.set(basename(file), file);
    }
  }
  const lines = logLines(raw).filter(line => isJob(line.job, lane, opts.e2eProfile));
  if (lines.some(l => /^##\[error\]/.test(l.text))) throw new Error(`${lane}: failed job log`);
  const byJob = new Map<string, LogLine[]>();
  for (const line of lines) {
    if (!byJob.has(line.job)) byJob.set(line.job, []);
    byJob.get(line.job)!.push(line);
  }
  if (opts.expectedJobs) {
    const expected = new Set(opts.expectedJobs);
    const missing = [...expected].filter(job => !byJob.has(job));
    const unexpected = [...byJob.keys()].filter(job => !expected.has(job));
    if (missing.length || unexpected.length) throw new Error(`${lane}: timing job set differs from run metadata (missing: ${missing.join(', ') || 'none'}; unexpected: ${unexpected.join(', ') || 'none'})`);
  }
  for (const [job, records] of byJob) {
    const captured = records.filter(line => line.step === 'capture' || line.text.startsWith('##[gbrain-capture-'));
    if (!captured.length && !opts.requireCapture) continue;
    if (!captured.length || captured[0]!.text !== '##[gbrain-capture-start]' ||
        captured.at(-1)!.text !== '##[gbrain-capture-complete] exit=0' ||
        captured.filter(line => line.text === '##[gbrain-capture-start]').length !== 1 ||
        captured.filter(line => line.text.startsWith('##[gbrain-capture-complete]')).length !== 1) {
      throw new Error(`${job}: missing or unsuccessful capture completion`);
    }
  }
  if (lane === "unit") {
    const events = parseLog(raw);
    for (const job of byJob.keys()) {
      const files = events.filter(e => e.job === job && e.file);
      const end = events.find(e => e.job === job && e.summaryCount !== undefined);
      if (!end || end.failures !== 0 || end.summaryCount !== files.length || !files.length) throw new Error(`${job}: missing, failed, or incomplete Bun summary`);
      const ordered = [...files, end];
      if (ordered.some((e, i) => i > 0 && e.timestampMs < ordered[i - 1]!.timestampMs)) throw new Error(`${job}: out-of-order timing records`);
      if (new Set(files.map(e => e.file)).size !== files.length) throw new Error(`${job}: duplicate file headers`);
    }
    const weights = computeWeights(events);
    if (!weights.size) throw new Error("unit: no complete timing data");
    return weights;
  }
  const duration = (text: string, multiplier: number, job: string) => {
    const value = Number(text) * multiplier;
    if (!Number.isFinite(value) || value < 0) throw new Error(`${job}: invalid duration ${text}`);
    return value;
  };
  const weights = new Map<string, number>();
  for (const [job, records] of byJob) {
    const found = new Map<string, number>();
    let expected: number | undefined, current: string | undefined;
    let failures: number | undefined;
    let completed: { duration: number; failures: number | undefined } | undefined;
    const finishFile = () => {
      if (!current || !completed || completed.failures !== 0 || found.has(current)) throw new Error(`${job}: missing, failed or duplicate file ${current}`);
      found.set(current, completed.duration);
      current = undefined;
      completed = undefined;
    };
    for (const { text } of records) {
      if (lane === "serial") {
        const pass = /^\[serial-tests\] PASS ([\d.]+)s (test\/\S+\.serial\.test\.ts)(?:\s|$)/.exec(text);
        if (pass) found.set(pass[2]!, Math.max(found.get(pass[2]!) ?? 0, duration(pass[1]!, 1, job)));
        const end = /^\[serial-tests\] all (\d+) file\(s\) passed/.exec(text);
        if (end) expected = Number(end[1]);
      } else {
        const start = /^=== ([^/]+\.test\.ts) ===$/.exec(text);
        if (start) {
          if (current) finishFile();
          if (expected !== undefined) throw new Error(`${job}: file after E2E completion`);
          current = full ? fullPaths.get(start[1]!) : `test/e2e/${start[1]}`;
          if (!current) throw new Error(`${job}: file outside the full source corpus: ${start[1]}`);
          failures = undefined;
        }
        const fail = /^\s*(\d+) fail\s*$/.exec(text);
        if (fail) failures = Number(fail[1]);
        const summary = /^Ran \d+ tests? across 1 file\. \[([\d.]+)(ms|s)\]/.exec(text);
        if (current && summary) {
          completed = { duration: duration(summary[1]!, summary[2] === "s" ? 1000 : 1, job), failures };
          failures = undefined;
        }
        const end = /^Files: (\d+) total, (\d+) passed, (\d+) failed$/.exec(text);
        if (end) {
          if (current) finishFile();
          if (Number(end[3]) !== 0 || end[1] !== end[2]) throw new Error(`${job}: failed or inconsistent E2E totals`);
          expected = Number(end[1]);
        }
        if (/^ERROR: HOME isolation breach/.test(text)) throw new Error(`${job}: isolation failure`);
      }
    }
    // Only the runner's explicit no-work sentinel permits a job without a
    // summary. Setup-only/truncated jobs must not disappear from the evidence.
    if (!full && lane === 'e2e' && !found.size && expected === undefined && !current &&
        records.some(line => line.text === 'selected E2E: explicit empty selection; no tests launched')) continue;
    if (current || expected === undefined || expected !== found.size) throw new Error(`${job}: incomplete ${lane} execution`);
    for (const [file, duration] of found) {
      if (full && weights.has(file)) throw new Error(`full E2E: duplicate execution across jobs: ${file}`);
      weights.set(file, Math.max(weights.get(file) ?? 0, duration));
    }
  }
  if (!weights.size) throw new Error(`${lane}: no complete timing data`);
  if (full) {
    const missing = opts.expectedFiles!.filter(file => !weights.has(file));
    if (missing.length) throw new Error(`full E2E: missing source files: ${missing.join(', ')}`);
  }
  return weights;
}
/** Each file's median over the runs that measured it (the mean of the middle two for an even count). */
export function medianWeights(runs: readonly Map<string, number>[]): Map<string, number> {
  const values = new Map<string, number[]>();
  for (const run of runs) for (const [file, value] of run) values.set(file, [...(values.get(file) ?? []), value]);
  return new Map([...values].map(([file, list]) => {
    const sorted = list.sort((a, b) => a - b);
    const mid = Math.floor(sorted.length / 2);
    return [file, sorted.length % 2 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2];
  }));
}
export function serializeWeights(weights: Map<string, number>): string {
  for (const [file, value] of weights) {
    if (!Number.isFinite(value) || value < 0) throw new Error(`invalid weight for ${file}: ${value}`);
  }
  return JSON.stringify(Object.fromEntries([...weights].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))), null, 2) + "\n";
}
function gh(args: string[]): string {
  const r = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 });
  if (r.status !== 0) throw new Error(`gh ${args.join(" ")} failed: ${r.stderr}`);
  return r.stdout;
}
/** The capture artifact a unit shard uploads (`timings-unit-N`, test.yml): its records carry the capture markers. */
function capturedUnitLog(run: string, job: string): string {
  const shard = /^test \((\d+)\)$/.exec(job)?.[1];
  if (!shard) throw new Error(`unit: unexpected job name ${job}`);
  const dir = mkdtempSync(join(tmpdir(), "gbrain-mine-unit-"));
  try {
    gh(["run", "download", run, "--name", `timings-unit-${shard}`, "--dir", dir]);
    return readFileSync(join(dir, "unit.log"), "utf8");
  } finally { rmSync(dir, { recursive: true, force: true }); }
}
function git(args: string[]): string {
  const r = spawnSync("git", args, { cwd: ROOT, encoding: "utf8", maxBuffer: 16 * 1024 * 1024 });
  if (r.status !== 0) throw new Error(`git ${args.join(" ")} failed: ${r.stderr}`);
  return r.stdout.trimEnd();
}
export function sourceE2ECorpus(commit: string): string[] {
  if (commit !== git(["rev-parse", "HEAD"])) throw new Error("full E2E source SHA differs from checkout HEAD");
  const tracked = git(["ls-tree", "-r", "--name-only", commit]).split("\n").filter(file => file.endsWith(".test.ts"));
  const root = mkdtempSync(join(tmpdir(), "gbrain-full-e2e-source-"));
  try {
    mkdirSync(join(root, "scripts/lib"), { recursive: true });
    writeFileSync(join(root, "scripts/run-e2e.sh"), git(["show", `${commit}:scripts/run-e2e.sh`]));
    writeFileSync(join(root, "scripts/lib/test-env.sh"), git(["show", `${commit}:scripts/lib/test-env.sh`]));
    writeFileSync(join(root, "scripts/e2e-live-key-only.txt"), git(["show", `${commit}:scripts/e2e-live-key-only.txt`]));
    for (const file of tracked) {
      mkdirSync(dirname(join(root, file)), { recursive: true });
      writeFileSync(join(root, file), "");
    }
    const r = spawnSync("bash", ["scripts/run-e2e.sh", "--dry-run-list"], {
      cwd: root, encoding: "utf8", env: { ...process.env, HOME: root, GBRAIN_HOME: root, SHARD: "", COVERAGE_DIR: "" },
    });
    if (r.status !== 0) throw new Error(`source E2E discovery failed: ${r.stderr}`);
    const files = r.stdout.trim().split("\n").filter(Boolean);
    if (!files.length || new Set(files).size !== files.length || files.some(file => !tracked.includes(file))) throw new Error("source E2E discovery contains missing or duplicate paths");
    return files;
  } finally { rmSync(root, { recursive: true, force: true }); }
}
async function main(): Promise<void> {
  let lane: Lane = "unit", input: string | undefined, out: string | undefined;
  const runs: string[] = [];
  let e2eProfile: E2EProfile = "selected";
  const argv = process.argv.slice(2);
  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i]!;
    if (arg === "--help" || arg === "-h") {
      console.log("usage: bun run weights:mine [--lane unit|serial|e2e] [--e2e-profile selected|full] [--run ID [--run ID ...] | --from-file PATH | <stdin>] [--out PATH]\nseveral --run values (any order) give each file its median over those runs; files git no longer tracks are dropped\nfull E2E requires --lane e2e, exactly one --run and a matching source checkout");
      return;
    }
    if (!["--lane", "--e2e-profile", "--run", "--run-id", "--from-file", "--out"].includes(arg) || !argv[i + 1] || argv[i + 1]!.startsWith("--")) throw new Error(`unknown/incomplete option: ${arg}`);
    const value = argv[++i]!;
    if (arg === "--lane") {
      if (!["unit", "serial", "e2e"].includes(value)) throw new Error(`invalid lane: ${value}`);
      lane = value as Lane;
    } else if (arg === "--e2e-profile") {
      if (value !== "selected" && value !== "full") throw new Error(`invalid E2E profile: ${value}`);
      e2eProfile = value;
    } else if (arg === "--run" || arg === "--run-id") runs.push(value);
    else if (arg === "--from-file") input = value;
    else out = resolve(value);
  }
  if (runs.length && input) throw new Error("choose --run or --from-file, not both");
  if (new Set(runs).size !== runs.length) throw new Error("each --run may be given once");
  const full = e2eProfile === "full";
  if (full && (lane !== "e2e" || runs.length !== 1)) throw new Error("full E2E requires --lane e2e and exactly one --run for verified source provenance");
  out ??= resolve(ROOT, `scripts/${OUTPUTS[lane]}.json`);
  type Mined = { run: string | null; commit: string | null; measured: Map<string, number>; attempt?: number; jobs: Array<{ name: string; databaseId: number }>; expectedFiles?: string[]; raw: string };
  const mineRun = (run: string): Mined => {
    const info = JSON.parse(gh(["run", "view", run, "--json", full ? "attempt,conclusion,headSha,jobs" : "conclusion,headSha,jobs"]));
    if (info.conclusion !== "success") throw new Error(`run ${run} is not successful`);
    const jobs = info.jobs.filter((j: { name: string }) => isJob(j.name, lane, e2eProfile));
    if (!jobs.length || jobs.some((j: { conclusion: string }) => j.conclusion !== "success")) throw new Error(`run ${run} has no complete ${lane} lane`);
    // A packed unit job captured each of its shards as job `test (N)`.
    const expectedJobs: string[] = lane === "unit" ? jobs.flatMap((j: { name: string }) => unitJobShards(j.name)!.map(shard => `test (${shard})`))
      : jobs.map((j: { name: string }) => j.name);
    let attempt: number | undefined, expectedFiles: string[] | undefined;
    let identified: Array<{ name: string; databaseId: number }> = [];
    if (lane === "serial" || full) {
      if ((full && (!Number.isInteger(info.attempt) || info.attempt < 1)) || jobs.some((j: { databaseId: number }) => !Number.isInteger(j.databaseId) || j.databaseId < 1)) throw new Error(full ? "full E2E: missing attempt or job identity" : "serial: missing job identity");
      identified = jobs.map(({ name, databaseId }: { name: string; databaseId: number }) => ({ name, databaseId }));
    }
    if (full) {
      attempt = info.attempt;
      expectedFiles = sourceE2ECorpus(info.headSha);
    }
    const raw = full ? identified.map(job => gh(["run", "view", run, "--attempt", String(attempt), "--job", String(job.databaseId), "--log"])).join("\n")
      : lane === "unit" ? expectedJobs.map(job => capturedUnitLog(run, job)).join("\n")
      : lane === "serial" ? identified.map(job => gh(["run", "view", run, "--job", String(job.databaseId), "--log"])).join("\n")
      : gh(["run", "view", run, "--log"]);
    const measured = mineWeights(raw, lane, { expectedJobs, e2eProfile, expectedFiles, requireCapture: lane === "unit" });
    return { run, commit: info.headSha, measured, attempt, jobs: full ? identified : [], expectedFiles, raw };
  };
  let mined: Mined[];
  if (runs.length) mined = runs.map(mineRun);
  else {
    const raw = input ? readFileSync(input, "utf8") : await new Response(Bun.stdin.stream()).text();
    mined = [{ run: null, commit: null, jobs: [], raw, measured: mineWeights(raw, lane, { e2eProfile }) }];
  }
  const { run, commit, attempt, jobs: fullJobs, expectedFiles, raw } = mined[0]!;
  // Downloaded artifacts/stdin may cover only one shard. Only an authoritative
  // complete GitHub unit/serial/full-E2E run replaces the full map; selected E2E always
  // merges because its executed corpus depends on the diff.
  const mergeExisting = !full && (!run || lane === "e2e");
  // An older run, or the existing map a partial source merges into, can name a file deleted or renamed since;
  // the checkout's tracked files decide what stays, whatever the --run order.
  const tracked = new Set(git(["ls-files"]).split("\n"));
  const measured = new Map([...medianWeights(mined.map(m => m.measured))].filter(([file]) => tracked.has(file)));
  const weights = new Map([...(mergeExisting && existsSync(out) ? loadWeights(out) : [])].filter(([file]) => tracked.has(file)));
  for (const [file, duration] of measured) weights.set(file, duration);
  const metadata = { lane, unit: lane === "serial" ? "seconds" : "milliseconds", run: run ?? null, commit, source: run ? "github" : input ? "file" : "stdin",
    ...(mined.length > 1 ? { runs, commits: mined.map(m => m.commit) } : {}), measuredFiles: measured.size, totalFiles: weights.size, mergeExisting,
    ...(full ? { e2eProfile, attempt, jobs: fullJobs, logSha256: createHash("sha256").update(raw).digest("hex"), corpusSha256: createHash("sha256").update([...expectedFiles!].sort().join("\n") + "\n").digest("hex") } : {}),
  };
  const temp = `${out}.${process.pid}.tmp`;
  try {
    writeFileSync(temp, serializeWeights(weights));
    renameSync(temp, out);
    writeFileSync(out.endsWith(".json") ? out.replace(/\.json$/, ".metadata.json") : `${out}.metadata.json`, JSON.stringify(metadata, null, 2) + "\n");
  } finally { rmSync(temp, { force: true }); }
  console.error(`[weights:mine] ${lane}: ${measured.size} measured${mined.length > 1 ? ` (median of ${mined.length} runs)` : ""}, ${weights.size} total (${metadata.unit}); ${out}`);
}
if (import.meta.main) main().catch(error => { console.error(`weights:mine: ${error.message}`); process.exitCode = 1; });