Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
19.9 KB
Raw Blame
// run-verify-parallel.test.ts — pin the dispatcher's contract.
//
// We can't easily fake the 20-check list inside the script (it's a static
// array), but we CAN verify:
//   1. --dry-list emits one line per check
//   2. Unknown args exit 2
//   3. The fast-path measurement claim — running it on a clean tree
//      completes faster than the sequential `&&`-chain would have
//   4. A made-up failing check propagates exit 1 with the named check
//      in the failure report (covered by overriding the CHECKS array
//      via a sibling temp script)
//
// (3) is a soft regression guard, not a hard timing assertion (CI runners
// vary). (4) is the load-bearing test: failure surfaces with name + log.

import { describe, expect, it } from "bun:test";
import { spawnSync } from "node:child_process";
import {
  copyFileSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  readdirSync,
  rmSync,
  symlinkSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

const SCRIPT = "scripts/run-verify-parallel.sh";

describe("run-verify-parallel.sh — CLI contract", () => {
  it("--dry-list emits one line per check, exit 0", () => {
    const r = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" });
    expect(r.status).toBe(0);
    const lines = r.stdout.trim().split("\n").filter(Boolean);
    // The exact count is allowed to grow; assert > 10 and that every line
    // looks like a script name (no whitespace, no quotes).
    expect(lines.length).toBeGreaterThan(10);
    for (const l of lines) {
      expect(l).toMatch(/^[a-z][a-z0-9:_-]+$/);
    }
  });

  it("includes the load-bearing privacy + jsonb + typecheck checks", () => {
    const r = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" });
    expect(r.status).toBe(0);
    const set = new Set(r.stdout.trim().split("\n"));
    expect(set.has("check:privacy")).toBe(true);
    expect(set.has("check:jsonb")).toBe(true);
    expect(set.has("typecheck")).toBe(true);
    expect(set.has("check:operations-filter-bypass")).toBe(true);
  });

  it("unknown arg exits 2 with usage error", () => {
    const r = spawnSync("bash", [SCRIPT, "--bogus"], { encoding: "utf8" });
    expect(r.status).toBe(2);
    expect(r.stderr).toContain("unknown arg");
    expect(r.stderr).toContain("usage:");
  });
});

describe("guard registration ⇒ execution coverage", () => {
  // guard-self-test.sh enforces that every scripts/check-* guard is
  // REGISTERED in guards-manifest.tsv, but nothing enforced that a
  // registered guard actually EXECUTES anywhere — five guards sat
  // registered-but-dead until the v0.45.x test/eval/CI pass. This closes
  // the loop: every manifest guard must be reachable from verify's CHECKS
  // (via a package.json script that invokes its file), or be explicitly
  // exempted HERE with the reason it runs elsewhere / deliberately not.
  const EXECUTION_EXEMPT: Record<string, string> = {
    "check-bun-test-timeout.sh":
      "runs directly as a test.yml verify-job step (not via CHECKS — avoids a package.json edit)",
    "check-bash32.sh":
      "needs Docker or a native bash 3.2: runs as a test.yml verify-job step (bash:3.2 image) and in macos-validation.yml (/bin/bash), never in verify",
    "check-test-discriminates.sh":
      "manual per-PR discrimination helper (#3665): reverts named source files and re-runs bun test to prove the test fails without the fix — cannot know which hunk is 'the fix' on an arbitrary diff, so deliberately not in verify/CI (manifest row says the same)",
  };

  it("every manifest guard is executed by verify or explicitly exempt", () => {
    const manifestLines = readFileSync("scripts/guards-manifest.tsv", "utf8")
      .split("\n")
      .filter((l) => l.trim() && !l.startsWith("#"));
    const guards = manifestLines.map((l) => l.split("\t")[0]!).filter(Boolean);
    expect(guards.length).toBeGreaterThan(30);

    const pkg = JSON.parse(readFileSync("package.json", "utf8")) as {
      scripts: Record<string, string>;
    };
    const dry = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" });
    expect(dry.status).toBe(0);
    const executed = new Set(dry.stdout.trim().split("\n"));

    const missing: string[] = [];
    for (const g of guards) {
      if (EXECUTION_EXEMPT[g]) continue;
      const invokingKeys = Object.entries(pkg.scripts)
        .filter(([, cmd]) => cmd.includes(`scripts/${g}`))
        .map(([key]) => key);
      const covered = invokingKeys.some((k) => executed.has(k));
      if (!covered) missing.push(g);
    }
    expect(missing).toEqual([]);
  });
});

describe("run-verify-parallel.sh — failure surfacing (synthetic dispatcher)", () => {
  // We can't inject a fake check into the real script without touching the
  // CHECKS array. Instead, we write a SMALLER synthetic dispatcher that
  // shares the same shape (background jobs, exit-file aggregation, log
  // tail on failure) and assert the surfacing contract on it. This proves
  // the design, leaving the real script tested via CLI contract above.
  //
  // Each "check" is written as its own executable file so bash array
  // expansion doesn't try to word-split shell strings (the natural
  // failure mode of expressing checks as bash command strings is that
  // spaces and quotes don't survive `"${CHECKS[@]}"`).

  function writeSynth(checkBodies: { name: string; body: string }[]): string {
    const dir = mkdtempSync(join(tmpdir(), "verify-synth-"));
    const checkPaths: string[] = [];
    for (const c of checkBodies) {
      const p = `${dir}/${c.name}.sh`;
      writeFileSync(p, `#!/usr/bin/env bash\n${c.body}\n`, { mode: 0o755 });
      checkPaths.push(p);
    }
    const dispatcher = `${dir}/dispatch.sh`;
    // CHECKS is an array of absolute paths to executable scripts.
    const arrLit = checkPaths.map((p) => `"${p}"`).join(" ");
    writeFileSync(
      dispatcher,
      `#!/usr/bin/env bash
set -uo pipefail
LOG_DIR=$(mktemp -d /tmp/verify-synth-log-XXXXXX)
trap 'rm -rf "$LOG_DIR"' EXIT
CHECKS=(${arrLit})
PIDS=()
NAMES=()
for c in "\${CHECKS[@]}"; do
  base=$(basename "$c" .sh)
  NAMES+=("$base")
  (
    "$c" > "$LOG_DIR/$base.log" 2>&1
    echo $? > "$LOG_DIR/$base.exit"
  ) &
  PIDS+=($!)
done
for p in "\${PIDS[@]}"; do wait "$p" 2>/dev/null || true; done
FAIL=0
FAILED_NAMES=""
for i in "\${!NAMES[@]}"; do
  n="\${NAMES[$i]}"
  rc=$(cat "$LOG_DIR/$n.exit")
  if [ "$rc" != "0" ]; then
    FAIL=$((FAIL+1))
    FAILED_NAMES="$FAILED_NAMES $n"
    echo "--- $n (rc=$rc) ---" >&2
    tail -10 "$LOG_DIR/$n.log" >&2
  fi
done
if [ "$FAIL" -gt 0 ]; then
  echo "Failed:$FAILED_NAMES" >&2
  exit 1
fi
exit 0
`,
      { mode: 0o755 },
    );
    return dispatcher;
  }

  function cleanup(dispatcher: string) {
    rmSync(dispatcher.replace(/\/dispatch\.sh$/, ""), { recursive: true, force: true });
  }

  it("all-pass: exit 0, no failure block", () => {
    const d = writeSynth([
      { name: "alpha", body: "exit 0" },
      { name: "beta", body: "exit 0" },
      { name: "gamma", body: "exit 0" },
    ]);
    try {
      const r = spawnSync("bash", [d], { encoding: "utf8" });
      expect(r.status).toBe(0);
      expect(r.stderr).not.toContain("Failed:");
    } finally {
      cleanup(d);
    }
  });

  it("one-fails: exit 1, failure block names the check + shows log tail", () => {
    const d = writeSynth([
      { name: "alpha", body: "exit 0" },
      {
        name: "beta",
        body: 'echo "synthetic failure detail line" >&2\nexit 7',
      },
      { name: "gamma", body: "exit 0" },
    ]);
    try {
      const r = spawnSync("bash", [d], { encoding: "utf8" });
      expect(r.status).toBe(1);
      expect(r.stderr).toContain("Failed: beta");
      expect(r.stderr).toContain("synthetic failure detail line");
      expect(r.stderr).toMatch(/--- beta \(rc=7\) ---/);
    } finally {
      cleanup(d);
    }
  });

  it("two-fail: both names appear in the Failed: list", () => {
    const d = writeSynth([
      { name: "alpha", body: "exit 0" },
      { name: "beta", body: "echo err-A >&2\nexit 1" },
      { name: "gamma", body: "echo err-B >&2\nexit 2" },
    ]);
    try {
      const r = spawnSync("bash", [d], { encoding: "utf8" });
      expect(r.status).toBe(1);
      expect(r.stderr).toContain("err-A");
      expect(r.stderr).toContain("err-B");
      expect(r.stderr).toMatch(/Failed:\s+beta\s+gamma/);
    } finally {
      cleanup(d);
    }
  });
});

describe("run-verify-parallel.sh — no-timeout-binary fallback rc capture (regression)", () => {
  // macOS ships no `timeout`; without brew coreutils (`gtimeout`) — stock
  // machines, minimal containers, restricted/sandboxed PATHs — the dispatcher
  // degrades to the bg-pid + sleep-watchdog branch.
  //
  // Regression pinned here: each check's sentinel .exit file must record the
  // exit code of the CHECK (read right after `wait $pid`), not of the
  // watchdog teardown. The watchdog subshell is killed with SIGTERM and so
  // reports 143; reading `$?` after the teardown stamped 143 into every
  // sentinel — verify reported pass=0 fail=<all> while every per-check log
  // said OK.
  //
  // Hermetic on any host: the script runs from a tempdir copy with `bun`
  // stubbed (checks complete instantly, no repo needed) and PATH set to a
  // curated symlink dir containing everything the script calls EXCEPT
  // gtimeout/timeout — forcing the fallback branch even where coreutils is
  // installed.

  function makeFallbackHarness(timeoutSecs = "5"): { root: string; env: Record<string, string> } {
    const root = mkdtempSync(join(tmpdir(), "verify-fallback-"));
    mkdirSync(join(root, "scripts", "lib"), { recursive: true });
    copyFileSync(SCRIPT, join(root, "scripts", "run-verify-parallel.sh"));
    // The dispatcher sources the shared runner lib — stage it too (E3).
    copyFileSync("scripts/lib/test-env.sh", join(root, "scripts", "lib", "test-env.sh"));

    const bin = join(root, "bin");
    mkdirSync(bin);
    // Everything the dispatcher and its subshells invoke, minus timeout bins.
    for (const tool of ["bash", "sh", "env", "dirname", "mktemp", "date", "sleep", "cat", "tail", "head", "rm", "mkdir", "pkill", "grep", "sed", "awk", "wc", "tr", "sort"]) {
      const p = Bun.which(tool);
      if (p) symlinkSync(p, join(bin, tool));
    }
    // `bun run <name>` stand-in: instant, prints OK, exits 7 for the check
    // named in $STUB_FAIL_CHECK (if any).
    writeFileSync(
      join(bin, "bun"),
      `#!/usr/bin/env bash
name="\${2:-}"
[ -n "\${STUB_LOG:-}" ] && echo "start $name" >> "$STUB_LOG"
if [ -n "\${STUB_SLOW_CHECK:-}" ] && [ "$name" = "\${STUB_SLOW_CHECK}" ]; then sleep 2; fi
[ -n "\${STUB_LOG:-}" ] && echo "end $name" >> "$STUB_LOG"
echo "stub check OK: $name"
if [ -n "\${STUB_SKIP_CHECK:-}" ] && [ "$name" = "\${STUB_SKIP_CHECK}" ]; then
  echo "GBRAIN_CHECK_SKIPPED: subject absent in this checkout"
fi
if [ -n "\${STUB_FAIL_CHECK:-}" ] && [ "$name" = "\${STUB_FAIL_CHECK}" ]; then
  echo "stub check failing: $name" >&2
  exit 7
fi
exit 0
`,
      { mode: 0o755 },
    );

    return {
      root,
      env: {
        PATH: bin,
        HOME: process.env.HOME ?? root,
        TMPDIR: process.env.TMPDIR ?? "/tmp",
        GBRAIN_VERIFY_TIMEOUT: timeoutSecs,
        GBRAIN_VERIFY_LOG_DIR: join(root, "logs"),
      },
    };
  }

  it("all checks passing → exit 0, every sentinel records 0 (not the watchdog's 143)", () => {
    const { root, env } = makeFallbackHarness();
    try {
      const r = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], { encoding: "utf8", env });
      expect(r.stderr).toMatch(/pass=\d+ fail=0/);
      expect(r.status).toBe(0);
      const exits = readdirSync(join(root, "logs")).filter((f) => f.endsWith(".exit"));
      expect(exits.length).toBeGreaterThan(10);
      for (const f of exits) {
        expect(readFileSync(join(root, "logs", f), "utf8").trim()).toBe("0");
      }
    } finally {
      rmSync(root, { recursive: true, force: true });
    }
  });

  it("records every check's real outcome: a self-skip is a skip (not a pass) in outcomes.tsv and the receipt", () => {
    const { root, env } = makeFallbackHarness();
    try {
      const receipts = join(root, "receipts");
      const r = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], {
        encoding: "utf8",
        env: { ...env, STUB_SKIP_CHECK: "check:grok-pin", STUB_FAIL_CHECK: "check:jsonb", GBRAIN_TEST_RECEIPT_DIR: receipts },
      });
      expect(r.status).toBe(1);
      expect(r.stderr).toMatch(/fail=1 skip=1\b/);
      expect(r.stderr).toContain("check:grok-pin: subject absent in this checkout");
      const outcomes = readFileSync(join(root, "logs", "outcomes.tsv"), "utf8");
      expect(outcomes).toContain("check:grok-pin\tskip\t0\tsubject absent in this checkout");
      expect(outcomes).toContain("check:jsonb\tfail\t7\t");
      expect(outcomes).toContain("check:privacy\tpass\t0\t");
      const junit = readFileSync(join(receipts, "verify--all--primary.junit.xml"), "utf8");
      expect(junit).toContain('<testcase name="check:grok-pin" classname="verify" file="verify"><skipped message="subject absent in this checkout" /></testcase>');
      expect(junit).toContain('<testcase name="check:jsonb" classname="verify" file="verify"><failure message="fail rc=7" /></testcase>');
      expect(readFileSync(join(receipts, "verify--all--primary.receipt"), "utf8")).toContain("exit=1");
    } finally {
      rmSync(root, { recursive: true, force: true });
    }
  });

  it("keeps its default temp log directory when a check fails and names it", () => {
    const { root, env } = makeFallbackHarness();
    const { GBRAIN_VERIFY_LOG_DIR: _unused, ...defaults } = env;
    try {
      const r = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], {
        encoding: "utf8",
        env: { ...defaults, STUB_FAIL_CHECK: "check:jsonb" },
      });
      expect(r.status).toBe(1);
      const kept = /per-check logs kept in (\S+)/.exec(r.stderr)?.[1];
      expect(kept).toBeDefined();
      expect(readFileSync(join(kept!, "check_jsonb.log"), "utf8")).toContain("stub check failing: check:jsonb");
      rmSync(kept!, { recursive: true, force: true });
    } finally {
      rmSync(root, { recursive: true, force: true });
    }
  });

  it("#9316: creates its default temp log directory under TMPDIR", () => {
    const { root, env } = makeFallbackHarness();
    const { GBRAIN_VERIFY_LOG_DIR: _unused, ...defaults } = env;
    const tmp = join(root, "tmp");
    mkdirSync(tmp);
    try {
      const r = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], {
        encoding: "utf8",
        env: { ...defaults, TMPDIR: tmp, STUB_FAIL_CHECK: "check:jsonb" },
      });
      expect(r.status).toBe(1);
      const kept = /per-check logs kept in (\S+)/.exec(r.stderr)?.[1];
      if (kept && !kept.startsWith(`${tmp}/`)) rmSync(kept, { recursive: true, force: true });
      expect(kept?.startsWith(`${tmp}/gbrain-verify-`)).toBe(true);
    } finally {
      rmSync(root, { recursive: true, force: true });
    }
  });

  it("a self-timed SOLO check starts only after every pool check has finished, and fails like any other", () => {
    const { root, env } = makeFallbackHarness();
    try {
      const log = join(root, "order.log");
      const dry = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh"), "--dry-list"], { encoding: "utf8", env }).stdout.trim().split("\n");
      expect(dry.at(-1)).toBe("check:guard-self-test");
      expect(dry.filter((c) => c === "check:guard-self-test")).toHaveLength(1);
      const r = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], {
        encoding: "utf8",
        env: { ...env, STUB_LOG: log, STUB_SLOW_CHECK: "typecheck", GBRAIN_VERIFY_MAX_PARALLEL: "4" },
      });
      expect(r.status, r.stderr).toBe(0);
      expect(r.stderr).toMatch(/running \d+ checks: \d+ in the pool, then 1 solo/);
      const checks = new Set(dry);
      const lines = readFileSync(log, "utf8").trim().split("\n").filter((l) => checks.has(l.replace(/^(start|end) /, "")));
      const soloStart = lines.indexOf("start check:guard-self-test");
      expect(soloStart).toBeGreaterThan(-1);
      const poolEnds = lines.filter((l) => l.startsWith("end ") && l !== "end check:guard-self-test");
      expect(poolEnds).toHaveLength(dry.length - 1);
      expect(lines.slice(soloStart).filter((l) => l !== "end check:guard-self-test")).toEqual(["start check:guard-self-test"]);
      expect(lines.indexOf("end typecheck")).toBeLessThan(soloStart);
      const failing = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], {
        encoding: "utf8",
        env: { ...env, STUB_FAIL_CHECK: "check:guard-self-test" },
      });
      expect(failing.status).toBe(1);
      expect(failing.stderr).toContain("--- check:guard-self-test (rc=7)");
      expect(readFileSync(join(root, "logs", "outcomes.tsv"), "utf8")).toMatch(/^check:guard-self-test\tfail\t7\t\S+\t\d+$/m);
    } finally {
      rmSync(root, { recursive: true, force: true });
    }
  }, 60_000);

  it("typecheck runs on every platform: only the macOS 26 workflow raises the per-check cap, Linux keeps 120 s (#6056)", () => {
    const workflows = readdirSync(".github/workflows").filter((f) => /\.ya?ml$/.test(f));
    const read = (f: string) => readFileSync(join(".github/workflows", f), "utf8");
    expect(workflows.filter((f) => read(f).includes("GBRAIN_VERIFY_TIMEOUT"))).toEqual(["macos-validation.yml"]);
    const mac = read("macos-validation.yml");
    expect(mac).toMatch(/^\s+runs-on: macos-26$/m);
    expect(mac).toMatch(/^\s+GBRAIN_VERIFY_TIMEOUT: '240'$/m);
    for (const f of workflows) expect(read(f)).not.toContain("TYPECHECK_COVERED_BY");
    expect(readFileSync(SCRIPT, "utf8")).not.toContain("TYPECHECK_COVERED_BY");
    expect(readFileSync(SCRIPT, "utf8")).toContain('TIMEOUT="${GBRAIN_VERIFY_TIMEOUT:-120}"');
    const testYml = read("test.yml");
    const verifyStart = testYml.indexOf("\n  verify:\n");
    const verifyJob = testYml.slice(verifyStart, testYml.indexOf("\n  serial-tests:\n", verifyStart));
    expect(verifyJob).toContain("- run: bun run verify");
    expect(verifyJob).toMatch(/runs-on: ubicloud-standard-\d+-ubuntu/);
    const dry = spawnSync("bash", [SCRIPT, "--dry-list"], { encoding: "utf8" }).stdout.split("\n");
    expect(dry).toContain("typecheck");
    const pkgScripts = (JSON.parse(readFileSync("package.json", "utf8")) as { scripts: Record<string, string> }).scripts;
    for (const [name, cmd] of Object.entries(pkgScripts)) expect(`${name}: ${cmd}`).not.toContain("GBRAIN_VERIFY_TIMEOUT");
  });

  it("one check failing → exit 1, sentinel records the check's own rc (7), not 143", () => {
    const { root, env } = makeFallbackHarness();
    try {
      const r = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], {
        encoding: "utf8",
        env: { ...env, STUB_FAIL_CHECK: "check:jsonb" },
      });
      expect(r.status).toBe(1);
      expect(r.stderr).toContain("--- check:jsonb (rc=7)");
      expect(r.stderr).toContain("stub check failing: check:jsonb");
      expect(r.stderr).toMatch(/fail=1\b/);
      expect(readFileSync(join(root, "logs", "check_jsonb.exit"), "utf8").trim()).toBe("7");
    } finally {
      rmSync(root, { recursive: true, force: true });
    }
  });

  it("the watchdog never outlives its check: no orphaned sleep holds the caller's pipes", () => {
    const timeoutSecs = "47";
    const { root, env } = makeFallbackHarness(timeoutSecs);
    const watchdogSleeps = () => spawnSync("pgrep", ["-f", `^sleep ${timeoutSecs}$`], { encoding: "utf8" }).stdout.trim().split("\n").filter(Boolean);
    try {
      const started = performance.now();
      const r = spawnSync("bash", [join(root, "scripts", "run-verify-parallel.sh")], { encoding: "utf8", env });
      expect(r.status).toBe(0);
      expect(performance.now() - started).toBeLessThan(Number(timeoutSecs) * 1000 / 2);
      if (Bun.which("pgrep")) expect(watchdogSleeps()).toEqual([]);
    } finally {
      for (const pid of watchdogSleeps()) try { process.kill(Number(pid)); } catch {}
      rmSync(root, { recursive: true, force: true });
    }
  }, 60_000);
});