Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.6 KB
Raw Blame
/**
 * scripts/contributor-audit.ts — the mechanical contributor audit.
 *
 * Protects: (1) the classification of each merged change and trial-merged PR
 * on the offline fixture range (scripts/contributor-audit-fixture.ts), driven
 * through the real CLI; (2) the scrubbed environment untrusted test code sees;
 * (3) resume against pinned PR heads; (4) the agent-contract refusals.
 * Regressions it catches: a whole-file revert or a skipped baseline changing a
 * verdict, an ambient DATABASE_URL or credential leaking into contributor code,
 * a resume mixing results from different PR heads.
 */
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { homedir, tmpdir } from 'node:os';
import { join } from 'node:path';
import { buildFixture } from '../../scripts/contributor-audit-fixture.ts';
import { classifyPath, classifyReversed, parseArgs, sandboxEnv } from '../../scripts/contributor-audit.ts';

const SCRIPT = join(import.meta.dir, '..', '..', 'scripts', 'contributor-audit.ts');

function audit(cwd: string, args: string[], env: Record<string, string | undefined> = {}) {
  const r = spawnSync(process.execPath, [SCRIPT, ...args], { cwd, encoding: 'utf8', env: { ...process.env, ...env }, timeout: 120_000 });
  return { code: r.status, stdout: r.stdout, stderr: r.stderr };
}

function withFixture(fn: (f: ReturnType<typeof buildFixture>) => void) {
  const dir = mkdtempSync(join(tmpdir(), 'contributor-audit-'));
  try { fn(buildFixture(join(dir, 'repo'))); } finally { rmSync(dir, { recursive: true, force: true }); }
}

describe('contributor audit on the fixture range', () => {
  test('classifies discriminating, non-discriminating, setup-failure, docs-only and conflicting changes, keeping later fixes in place', () => {
    withFixture(f => {
      const r = audit(f.dir, [`${f.base}..${f.head}`, '--prs', f.prsManifest, '--json', '--skip-security', '--skip-lanes']);
      expect(r.code, r.stderr).toBe(1);
      const report = JSON.parse(r.stdout);
      const byPr = Object.fromEntries(Object.values(report.cases as Record<string, any>).map(c => [c.pr, c]));
      expect(byPr[101].result).toBe('discriminates');
      expect(byPr[101].runs.baseline).toMatchObject({ exit: 0, pass: 1, fail: 0 });
      expect(byPr[101].runs.reversed).toMatchObject({ pass: 0, fail: 1 });
      expect(byPr[101].runs.restored).toMatchObject({ exit: 0, pass: 1, fail: 0 });
      expect(byPr[101].tests).toEqual(['test/clamp.test.ts']);
      expect(byPr[102].result).toBe('does_not_discriminate');
      expect(byPr[103]).toMatchObject({ result: 'setup_failed', reason: 'baseline_red' });
      expect(byPr[104]).toMatchObject({ result: 'not_audited', reason: 'no_product_change' });
      expect(byPr[106]).toMatchObject({ result: 'does_not_discriminate', audited_at: f.head });
      expect(byPr[107]).toMatchObject({ result: 'not_audited', reason: 'no_tests', subject: 'Fix score (#107)', author: 'hana-example' });
      expect(byPr[105].result).toBe('conflict');
      expect(byPr[105].reason).toContain('src/clamp.ts');
      expect(report.exit_code).toBe(1);

      const pinned = JSON.parse(readFileSync(join(report.run_dir, 'prs.pinned.json'), 'utf8'));
      expect(pinned.prs[0].head).toMatch(/^[0-9a-f]{40}$/);
      expect(pinned.base).toBe(f.base);

      const md = audit(f.dir, [`${f.base}..${f.head}`, '--resume', '--prs', join(report.run_dir, 'prs.pinned.json'), '--skip-security', '--skip-lanes']);
      expect(md.code).toBe(1);
      expect(md.stderr).toContain('resuming: 7 case(s) already done');
      expect(md.stdout).toContain('| **discriminates** |');
      expect(md.stdout).toContain('| Human verdict |');
    });
  });

  test('resume refuses when the PR pins differ from the saved run', () => {
    withFixture(f => {
      const range = `${f.base}..${f.head}`;
      expect(audit(f.dir, [range, '--prs', f.prsManifest, '--skip-security', '--skip-lanes']).code).toBe(1);
      const other = join(f.dir, 'other.json');
      writeFileSync(other, JSON.stringify({ prs: [{ head: f.base }] }));
      const r = audit(f.dir, [range, '--prs', other, '--resume', '--json', '--skip-security', '--skip-lanes']);
      expect(r.code).toBe(2);
      const env = JSON.parse(r.stdout);
      expect(env.code).toBe('preview_changed');
      expect(env.fix.argv).toContain('--resume');
    });
  });

  test('human verdicts print in their own column and never replace the mechanical result', () => {
    withFixture(f => {
      const verdicts = join(f.dir, 'verdicts.json');
      const first = audit(f.dir, [`${f.base}..${f.head}`, '--json', '--skip-security', '--skip-lanes']);
      const id = Object.values(JSON.parse(first.stdout).cases as Record<string, any>).find(c => c.pr === 102).id;
      writeFileSync(verdicts, JSON.stringify({ [id]: { verdict: 'rework', note: 'test never calls greet' } }));
      const r = audit(f.dir, [`${f.base}..${f.head}`, '--resume', '--verdicts', verdicts, '--skip-security', '--skip-lanes']);
      expect(r.stdout).toContain('| **does_not_discriminate** |');
      expect(r.stdout).toContain('| rework: test never calls greet |');
      writeFileSync(verdicts, JSON.stringify({ [id]: { verdict: 'lgtm' } }));
      const bad = audit(f.dir, [`${f.base}..${f.head}`, '--resume', '--verdicts', verdicts, '--json', '--skip-security', '--skip-lanes']);
      expect(bad.code).toBe(2);
      expect(JSON.parse(bad.stdout).code).toBe('invalid_params');
    });
  });
});

describe('untrusted code runs in a scrubbed environment', () => {
  test('the audited tests see no ambient database URL, credential or real HOME', () => {
    const dir = mkdtempSync(join(tmpdir(), 'contributor-audit-env-'));
    try {
      const repo = join(dir, 'repo');
      const probe = join(dir, 'probe.json');
      const git = (...a: string[]) => spawnSync('git', a, { cwd: repo, encoding: 'utf8' });
      spawnSync('mkdir', ['-p', join(repo, 'src'), join(repo, 'test')]);
      git('init', '-q', '-b', 'master');
      git('config', 'user.name', 't'); git('config', 'user.email', '[email protected]'); git('config', 'commit.gpgsign', 'false');
      writeFileSync(join(repo, 'src/x.ts'), 'export const x = 1;\n');
      git('add', '-A'); git('commit', '-qm', 'base');
      git('checkout', '-qb', 'probe');
      writeFileSync(join(repo, 'src/x.ts'), 'export const x = 2;\n');
      writeFileSync(join(repo, 'test/x.test.ts'), [
        "import { expect, test } from 'bun:test';",
        "import { readdirSync, writeFileSync } from 'node:fs';",
        "import { homedir } from 'node:os';",
        "import { x } from '../src/x';",
        `test('probe', () => { writeFileSync(${JSON.stringify(probe)}, JSON.stringify({ env: process.env, home: readdirSync(homedir()) })); expect(x).toBe(2); });`,
      ].join('\n') + '\n');
      git('add', '-A'); git('commit', '-qm', 'probe');
      git('checkout', '-q', 'master');
      git('merge', '-q', '--no-ff', 'probe', '-m', 'Merge pull request #7 from alice-example/probe');

      const r = audit(repo, ['HEAD~1..HEAD', '--json', '--skip-security', '--skip-lanes'], {
        DATABASE_URL: 'postgres://u:[email protected]/brain',
        GBRAIN_DATABASE_URL: 'postgres://u:[email protected]/brain',
        OPENAI_API_KEY: 'sk-fake-ambient', ANTHROPIC_API_KEY: 'sk-ant-fake', GH_TOKEN: 'ghp_fake', SSH_AUTH_SOCK: '/tmp/agent.sock',
      });
      expect(r.code, r.stderr).toBe(0);
      const seen = JSON.parse(readFileSync(probe, 'utf8')) as { env: Record<string, string>; home: string[] };
      expect(seen.env.DATABASE_URL ?? '').toBe('');
      expect(seen.env.GBRAIN_DATABASE_URL ?? '').toBe('');
      for (const key of ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'GH_TOKEN', 'SSH_AUTH_SOCK']) expect(seen.env[key]).toBeUndefined();
      expect(seen.env.HOME).not.toBe(homedir());
      expect(seen.env.GBRAIN_HOME.startsWith(seen.env.HOME)).toBe(true);
      for (const cred of ['.netrc', '.npmrc', '.git-credentials', '.ssh', '.gbrain', '.config']) expect(seen.home).not.toContain(cred);
      const allowed = new Set(Object.keys(sandboxEnv({ home: '/h', bunPath: process.execPath, cacheDir: '/c' })));
      const unexpected = Object.keys(seen.env).filter(k => !allowed.has(k) && !k.startsWith('BUN_') && !['_', 'SHLVL', 'PWD', 'OLDPWD', 'NODE_ENV'].includes(k));
      expect(unexpected).toEqual([]);
    } finally { rmSync(dir, { recursive: true, force: true }); }
  });

  test('--postgres is the only way a database URL reaches the tests, and it must be test-shaped', () => {
    const env = sandboxEnv({ home: '/h', bunPath: '/x/bun', cacheDir: '/c', postgresUrl: 'postgres://postgres:postgres@localhost:5435/gbrain_test' });
    expect(env.DATABASE_URL).toBe('postgres://postgres:postgres@localhost:5435/gbrain_test');
    expect(env.GBRAIN_DATABASE_URL).toBe('');
    expect(env.GBRAIN_TEST_ALLOW_DATABASE_URL).toBe('1');
    expect(() => parseArgs(['a..b', '--postgres', 'postgres://u:[email protected]/brain'])).toThrow(/test-shaped/);
  });
});

describe('classification and refusals', () => {
  test('the reversed run maps onto the discrimination vocabulary', () => {
    expect(classifyReversed({ exit: 1, pass: 0, fail: 2, timedOut: false }).result).toBe('discriminates');
    expect(classifyReversed({ exit: 0, pass: 3, fail: 0, timedOut: false }).result).toBe('does_not_discriminate');
    expect(classifyReversed({ exit: 1, pass: 0, fail: 0, timedOut: false })).toEqual({ result: 'vacuous_failure', reason: 'no_test_failed' });
    expect(classifyReversed({ exit: 128, pass: 0, fail: 0, timedOut: true })).toEqual({ result: 'vacuous_failure', reason: 'timeout' });
    expect(['test/a.test.ts', 'docs/x.md', 'src/a.ts', 'CHANGELOG.md', 'src/a.spec.ts'].map(classifyPath)).toEqual(['test', 'doc', 'product', 'doc', 'test']);
  });

  test('usage and preflight refusals are agent-contract envelopes with exit 2', () => {
    withFixture(f => {
      const unknown = audit(f.dir, [`${f.base}..${f.head}`, '--frobnicate', '--json']);
      expect(unknown.code).toBe(2);
      expect(JSON.parse(unknown.stdout)).toMatchObject({ code: 'unknown_flag', fix: { argv: ['bun', 'run', 'audit:contributors', '--help'] } });
      const badRef = audit(f.dir, ['nope..HEAD', '--json', '--skip-security']);
      expect(badRef.code).toBe(2);
      const env = JSON.parse(badRef.stdout);
      expect(env.code).toBe('invalid_params');
      expect(env.why).toContain('"nope" is not a commit');
      expect(env.fix.argv).toEqual(['git', 'fetch', 'origin']);
      const noRange = audit(f.dir, ['--json']);
      expect(JSON.parse(noRange.stdout).code).toBe('invalid_params');
      expect(existsSync(join(f.dir, '.git', 'contributor-audit'))).toBe(false);
    });
  });
});