Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.4 KB
Raw Blame
/**
 * Regression test: scripts/test-shard.sh exclusion + balance contract.
 *
 * Pins two invariants of the CI matrix shard script:
 *
 *  1. EXCLUDE *.serial.test.ts and test/e2e/ from every shard. Serial
 *     files share file-wide state (top-level mock.module, module
 *     singletons) that leaks across files in the same `bun test` shard
 *     process. Before v0.31.4.1 they were hashed into the same buckets
 *     as parallel files, which broke the quarantine —
 *     `eval-takes-quality-runner.serial.test.ts` stubbed `gateway.ts`
 *     and broke every `gateway.embedMultimodal` test in
 *     `voyage-multimodal.test.ts` on shard 2.
 *
 *  2. INCLUDE *.slow.test.ts. CI is the only default place slow files
 *     run; the local fast loop excludes them via run-unit-shard.sh. See
 *     CLAUDE.md "CI vs local: intentionally divergent file sets".
 *
 *  3. LPT balance: file counts across shards are within 5% of each other
 *     under the default (no-weights / cold-start) configuration. Once
 *     test-weights.json is populated, weighted balance also lands the
 *     imbalance ratio ≤ 1.5.
 *
 * Without (1), the v0.31.4.1 mock.module leak comes back. Without (2),
 * slow tests stop running in CI (silent regression — they don't fail,
 * they just never execute). Without (3), shard 3 will drift back to
 * 26-min p100 wallclock.
 */

import { describe, it, expect, beforeAll } from 'bun:test';
import { execFileSync } from 'child_process';
import { resolve } from 'path';
import { UNIT_SHARDS } from '../../scripts/sharding.ts';

const REPO_ROOT = resolve(import.meta.dir, '..', '..');
const SHARD_SH = resolve(REPO_ROOT, 'scripts/test-shard.sh');

// LPT partition via sharding.ts is ~30ms per shard (much faster than the
// pure-bash FNV-1a it replaced). Cache results so repeated assertions
// don't shell out per case.
const shardCache: Record<string, string[]> = {};

function dryRunList(shard: number, total: number): string[] {
  const key = `${shard}/${total}`;
  if (shardCache[key]) return shardCache[key];
  const out = execFileSync('bash', [SHARD_SH, '--dry-run-list', String(shard), String(total)], {
    cwd: REPO_ROOT,
    encoding: 'utf-8',
  });
  shardCache[key] = out.split('\n').map(s => s.trim()).filter(Boolean);
  return shardCache[key];
}

describe('test-shard.sh — exclusion contract', () => {
  beforeAll(() => {
    for (const shard of [1, 2, 3, 4]) dryRunList(shard, 4);
  }, 60_000);

  it('includes plain *.test.ts files in at least one shard', () => {
    const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4));
    expect(allFiles.length).toBeGreaterThan(0);
    expect(allFiles.some(f => /\.test\.ts$/.test(f) && !/\.serial\.test\.ts$/.test(f))).toBe(true);
  });

  it('excludes every *.serial.test.ts file from every shard', () => {
    for (const shard of [1, 2, 3, 4]) {
      const files = dryRunList(shard, 4);
      const leaks = files.filter(f => /\.serial\.test\.ts$/.test(f));
      expect(leaks, `shard ${shard} contains serial files`).toEqual([]);
    }
  });

  it('excludes the test/e2e/ subtree from every shard', () => {
    for (const shard of [1, 2, 3, 4]) {
      const files = dryRunList(shard, 4);
      const leaks = files.filter(f => f.startsWith('test/e2e/'));
      expect(leaks, `shard ${shard} contains e2e files`).toEqual([]);
    }
  });

  it('leaves the dedicated entity-card performance gate out of the unit matrix', () => {
    const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4));
    expect(allFiles).not.toContain('test/entity-card-perf.slow.test.ts');
    const fs = require('fs');
    const yaml = require('js-yaml');
    const workflow = yaml.load(fs.readFileSync(resolve(REPO_ROOT, '.github/workflows/test.yml'), 'utf8'));
    expect(workflow.jobs['slow-entity-resolve-perf'].steps.some((step: { run?: string }) =>
      step.run?.includes('bun test test/entity-card-perf.slow.test.ts'))).toBe(true);
    expect(workflow.jobs['test-status'].needs).toContain('slow-entity-resolve-perf');
  });

  it('runs export-scale and reconcile-crash exactly once per event outside the unit matrix', () => {
    const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4));
    expect(allFiles).not.toContain('test/export-scale.slow.test.ts');
    const reconcileCrash = require('fs').readdirSync(resolve(REPO_ROOT, 'test')).filter((f: string) => /^reconcile-crash-.*\.slow\.test\.ts$/.test(f));
    expect(reconcileCrash.length).toBe(4);
    for (const file of reconcileCrash) expect(allFiles).not.toContain(`test/${file}`);
    const fs = require('fs');
    const yaml = require('js-yaml');
    const workflow = yaml.load(fs.readFileSync(resolve(REPO_ROOT, '.github/workflows/test.yml'), 'utf8'));
    const exportStep = workflow.jobs['slow-entity-resolve-perf'].steps.find((step: { run?: string }) =>
      step.run?.includes('bun test test/export-scale.slow.test.ts'));
    expect(exportStep.env.GBRAIN_TEST_EXPORT_SCALE_PAGES).toContain("'100001'");
    const persistence = yaml.load(fs.readFileSync(resolve(REPO_ROOT, '.github/workflows/persistence-validation.yml'), 'utf8'));
    expect(persistence.jobs.reconciliation.strategy.matrix.engine).toContain('pglite');
    expect(persistence.jobs.reconciliation.steps.some((step: { run?: string }) =>
      step.run?.includes('test/reconcile-crash-*.slow.test.ts'))).toBe(true);
  });

  it('INCLUDES *.slow.test.ts files (CI matrix is where slow files run)', () => {
    const allFiles = [1, 2, 3, 4].flatMap(s => dryRunList(s, 4));
    const slowFiles = allFiles.filter(f => /\.slow\.test\.ts$/.test(f));
    // The repo currently has 3 slow files — pin >= 1 so the test stays
    // green if the count changes but fails loud if slow files vanish
    // entirely (regression: someone re-added -not -name '*.slow.test.ts'
    // to the find clause).
    expect(slowFiles.length).toBeGreaterThan(0);
  });

  it('partitions every file across shards without overlap', () => {
    const seen = new Map<string, number>();
    for (const shard of [1, 2, 3, 4]) {
      for (const f of dryRunList(shard, 4)) {
        if (seen.has(f)) {
          throw new Error(`file ${f} appears in shard ${seen.get(f)} AND shard ${shard}`);
        }
        seen.set(f, shard);
      }
    }
    expect(seen.size).toBeGreaterThan(0);
  });
});

describe('test-shard.sh — LPT balance contract', () => {
  // LPT balances WALLCLOCK (sum of weights), not file count. When real
  // weights are loaded from scripts/test-weights.json, shards with
  // heavier files have FEWER files — that's the optimization working.
  // We assert wallclock balance via the imbalance ratio (≤1.5).

  // Read weights from the committed JSON. If it doesn't exist yet (early
  // in the wave) or is empty, we degrade to cold-start round-robin and
  // assert file-count balance instead.
  let weightsMap = new Map<string, number>();
  let weightsLoaded = false;
  beforeAll(() => {
    try {
      const fs = require('fs');
      const path = require('path');
      const p = path.resolve(REPO_ROOT, 'scripts/test-weights.json');
      if (fs.existsSync(p)) {
        const raw = JSON.parse(fs.readFileSync(p, 'utf8'));
        for (const [k, v] of Object.entries(raw)) {
          if (typeof v === 'number' && Number.isFinite(v)) weightsMap.set(k, v);
        }
        weightsLoaded = weightsMap.size > 0;
      }
    } catch {
      // fall through to cold-start mode
    }
  });

  function totalsFor(shards: string[][]): number[] {
    // p75 fallback mirroring sharding.ts's computeQuantile choice (the
    // weight distribution is right-skewed and missing files skew heavy).
    const sorted = Array.from(weightsMap.values()).sort((a, b) => a - b);
    const fallback = weightsLoaded
      ? sorted[Math.min(sorted.length - 1, Math.ceil(0.75 * sorted.length) - 1)] ?? 30
      : 1;
    return shards.map((s) =>
      s.reduce((acc, f) => acc + (weightsMap.get(f) ?? fallback), 0),
    );
  }

  // CI runs THIS many shards (test.yml matrix). The old version of this
  // test asserted 4- and 6-shard splits — configurations nothing runs.
  const CI_SHARDS = UNIT_SHARDS;

  it('CI_SHARDS matches the test.yml matrix (parsed, not regexed)', () => {
    const fs = require('fs');
    const yaml = require('js-yaml');
    const wf = yaml.load(
      fs.readFileSync(resolve(REPO_ROOT, '.github/workflows/test.yml'), 'utf8'),
    ) as { jobs: { test: { strategy: { matrix: { shards: number[][] } } } } };
    // Each job runs a pack of shards (scripts/test-shard-pack.sh); together they are 1..CI_SHARDS.
    const packs = wf.jobs.test.strategy.matrix.shards;
    expect(Array.isArray(packs)).toBe(true);
    expect(packs.flat()).toEqual(Array.from({ length: CI_SHARDS }, (_, i) => i + 1));
  });

  it(`${CI_SHARDS}-shard wallclock imbalance ratio ≤ 1.5 (the configuration CI actually runs)`, () => {
    const shards = Array.from({ length: CI_SHARDS }, (_, i) => dryRunList(i + 1, CI_SHARDS));
    for (const s of shards) expect(s.length).toBeGreaterThan(0);
    const totals = totalsFor(shards);
    const ratio = Math.max(...totals) / Math.min(...totals);
    expect(ratio).toBeLessThanOrEqual(1.5);
  }, 60_000);

  // The two guards below are what make the ratio assertion above MEAN
  // something: recomputing totals with the same weights the partitioner
  // used is near-tautological — unless the weights actually cover the
  // corpus and refer to real files. Weight rot (files added without a
  // re-mine, or renamed away from their entries) used to be invisible:
  // 45% of the corpus once rode a 30ms median fallback while really
  // averaging ~4s, and the "balanced" partition was balanced on fiction.
  it('weights cover ≥70% of matrix-eligible files (anti-rot gate)', () => {
    const all = Array.from(
      new Set(Array.from({ length: CI_SHARDS }, (_, i) => dryRunList(i + 1, CI_SHARDS)).flat()),
    );
    expect(all.length).toBeGreaterThan(0);
    const covered = all.filter((f) => weightsMap.has(f)).length;
    const coverage = covered / all.length;
    // Regenerate from the latest green Test run:
    //   bun run scripts/mine-shard-weights.ts --run <run id>
    expect(coverage).toBeGreaterThanOrEqual(0.7);
  }, 60_000);

  it('no stale weight keys — every entry names a tracked file', () => {
    const tracked = new Set(
      execFileSync('git', ['ls-files', 'test', 'evals'], { cwd: REPO_ROOT, encoding: 'utf-8' })
        .split('\n')
        .map((s) => s.trim())
        .filter(Boolean),
    );
    const stale = [...weightsMap.keys()].filter((k) => !tracked.has(k));
    expect(stale).toEqual([]);
  });

  it('6-shard partition is deterministic across runs', () => {
    // Clear cache + re-run for one shard, compare to the cached result.
    const cached = [...dryRunList(1, 6)];
    delete shardCache['1/6'];
    const fresh = dryRunList(1, 6);
    expect(fresh).toEqual(cached);
  });
});

describe('test-shard.sh — execution order contract', () => {
  // Bun treats a bare `test/x.test.ts` argument as a name filter and runs
  // the matches in directory-scan order, so the planned LPT order never
  // reached the shard process and cross-file pollution depended on the
  // runner's filesystem. Drive the real script with a fake partitioner
  // that reverses the sorted list, and the real Bun on tiny fixture files.
  it('runs the shard files in the planned order', () => {
    const { chmodSync, copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } = require('fs');
    const { tmpdir } = require('os');
    const { join } = require('path');
    const root = mkdtempSync(join(tmpdir(), 'gbrain-test-shard-order-'));
    try {
      for (const dir of ['scripts/lib', 'test', 'evals', 'bin']) mkdirSync(join(root, dir), { recursive: true });
      copyFileSync(SHARD_SH, join(root, 'scripts/test-shard.sh'));
      copyFileSync(resolve(REPO_ROOT, 'scripts/lib/test-env.sh'), join(root, 'scripts/lib/test-env.sh'));
      const names = ['alpha', 'bravo', 'charlie', 'delta', 'echo', 'foxtrot'];
      for (const name of names) {
        writeFileSync(join(root, `test/${name}.test.ts`),
          `import { test } from 'bun:test';\nimport { appendFileSync } from 'fs';\n` +
          `test('${name}', () => { appendFileSync(process.env.ORDER_LOG!, '${name}\\n'); });\n`);
      }
      writeFileSync(join(root, 'bin/bun'), `#!/usr/bin/env bash
set -eu
if [ "$1" = run ] && [ "$2" = scripts/sharding.ts ]; then
  awk '{ lines[NR] = $0 } END { for (i = NR; i > 0; i--) print lines[i] }'
  exit 0
fi
[ "$1" = test ] || exit 1
printf '%s\\n' "$@" > "$ARGV_LOG"
exec "$REAL_BUN" "$@"
`);
      chmodSync(join(root, 'bin/bun'), 0o755);
      const env = {
        ...process.env,
        PATH: `${join(root, 'bin')}:${process.env.PATH}`,
        REAL_BUN: process.execPath,
        ORDER_LOG: join(root, 'order.log'),
        ARGV_LOG: join(root, 'argv.log'),
        GBRAIN_NO_SNAPSHOT: '1',
        GBRAIN_TEST_RECEIPT_DIR: join(root, 'receipts'),
        COVERAGE_DIR: '',
      };
      const planned = execFileSync('bash', [join(root, 'scripts/test-shard.sh'), '--dry-run-list', '1', '1'], { cwd: root, encoding: 'utf-8', env })
        .split('\n').filter(Boolean);
      expect(planned).toEqual([...names].reverse().map(n => `test/${n}.test.ts`));
      execFileSync('bash', [join(root, 'scripts/test-shard.sh'), '1', '1'], { cwd: root, encoding: 'utf-8', env, stdio: 'pipe' });
      const argv = readFileSync(join(root, 'argv.log'), 'utf-8').split('\n').filter((a: string) => a.endsWith('.test.ts'));
      expect(argv).toEqual(planned.map(f => `./${f}`));
      expect(readFileSync(join(root, 'order.log'), 'utf-8').split('\n').filter(Boolean)).toEqual([...names].reverse());
      // Receipt mode forces the single-invocation xargs tripwire, and the
      // receipt `.files` plus Bun's JUnit report keep the canonical bare
      // names in planned order, so the ledger matches assigned to executed.
      const rawArgv = readFileSync(join(root, 'argv.log'), 'utf-8').split('\n');
      expect(rawArgv.some((a: string) => a.startsWith('--reporter-outfile='))).toBe(true);
      const receiptDir = join(root, 'receipts');
      const id = 'unit--s1of1--primary';
      expect(readFileSync(join(receiptDir, `${id}.files`), 'utf-8').split('\n').filter(Boolean)).toEqual(planned);
      expect(readFileSync(join(receiptDir, `${id}.receipt`), 'utf-8')).toContain('exit=0');
      const junit = readFileSync(join(receiptDir, `${id}.junit.xml`), 'utf-8');
      const suites = [...junit.matchAll(/<testsuite name="([^"]+)"[^>]*file="([^"]+)"/g)].map(m => m[2]);
      expect(suites).toEqual(planned);
      expect([...junit.matchAll(/<testcase /g)]).toHaveLength(names.length);
    } finally {
      rmSync(root, { recursive: true, force: true });
    }
  }, 60_000);
});