Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
16.6 KB
Raw Blame
/**
 * Fixture-driven unit tests for scripts/check-test-isolation.sh.
 *
 * Spawns the script in a tmpdir with hand-crafted fake test files and
 * asserts the lint's exit code + violation messages match expectations.
 * No env mutation, no mock.module, no PGLite — this test file is itself
 * subject to the lint (it ships outside *.serial.test.ts and outside
 * test/e2e/).
 */

import { describe, it, expect } from 'bun:test';
import { spawnSync } from 'child_process';
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from 'fs';
import { tmpdir } from 'os';
import { join, resolve } from 'path';

const REPO_ROOT = resolve(import.meta.dir, '..', '..');
const LINT_SH = resolve(REPO_ROOT, 'scripts/check-test-isolation.sh');

interface FakeFile {
  /** Path relative to the tmpdir's `test/` directory. */
  path: string;
  contents: string;
}

interface RunResult {
  status: number;
  stdout: string;
  stderr: string;
}

function runLintIn(files: FakeFile[], allowlist: string[] = [], env: Record<string, string> = {}): RunResult {
  const dir = mkdtempSync(join(tmpdir(), 'lint-isolation-'));
  mkdirSync(join(dir, 'test'), { recursive: true });
  mkdirSync(join(dir, 'scripts'), { recursive: true });

  for (const f of files) {
    const full = join(dir, 'test', f.path);
    mkdirSync(resolve(full, '..'), { recursive: true });
    writeFileSync(full, f.contents);
  }
  // Empty allowlist file ensures the script reads OUR allowlist, not the
  // real repo's, regardless of git toplevel resolution.
  writeFileSync(
    join(dir, 'scripts/check-test-isolation.allowlist'),
    allowlist.length > 0 ? allowlist.join('\n') + '\n' : '',
  );

  const r = spawnSync('bash', [LINT_SH, 'test'], {
    cwd: dir,
    encoding: 'utf-8',
    env: { ...process.env, ...env },
  });
  return { status: r.status ?? -1, stdout: r.stdout, stderr: r.stderr };
}

describe('check-test-isolation.sh', () => {
  describe('clean files', () => {
    it('returns 0 when no test files violate any rule', () => {
      const r = runLintIn([
        {
          path: 'a.test.ts',
          contents: `import { test, expect } from 'bun:test';\ntest('ok', () => expect(1).toBe(1));\n`,
        },
      ]);
      expect(r.status).toBe(0);
      expect(r.stdout).toContain('check-test-isolation: OK');
    });

    it('batches candidate scans instead of starting grep for every clean file', () => {
      const bin = mkdtempSync(join(tmpdir(), 'lint-grep-count-'));
      const log = join(bin, 'calls');
      writeFileSync(join(bin, 'grep'), `#!/usr/bin/env bash\nprintf 'grep\\n' >> "$GREP_CALL_LOG"\nexec "$REAL_GREP" "$@"\n`, { mode: 0o755 });
      try {
        const files = Array.from({ length: 260 }, (_, i) => ({ path: `clean ${i}.test.ts`, contents: 'if (process.env.TEST_MODE === "yes") {}\n' }));
        const env = { PATH: `${bin}:${process.env.PATH}`, GREP_CALL_LOG: log, REAL_GREP: Bun.which('grep')! };
        const r = runLintIn(
          files,
          [],
          env,
        );
        expect(r.status).toBe(0);
        expect(r.stdout).toContain('260 non-serial unit files scanned');
        expect(readFileSync(log, 'utf8').trim().split('\n').length).toBeLessThan(10);
        const bad = runLintIn([...files, { path: 'z bad.test.ts', contents: `Reflect.set(process.env, 'OOPS', 'bad');\n` }], [], env);
        expect(bad.status).toBe(1);
        expect(bad.stdout).toContain('z bad.test.ts');
        expect(bad.stdout).toContain('rule R1');
      } finally {
        rmSync(bin, { recursive: true, force: true });
      }
    });
    it('lints candidate files in one pass instead of starting grep or awk per file (#8053)', () => {
      const bin = mkdtempSync(join(tmpdir(), 'lint-candidate-count-'));
      const log = join(bin, 'calls');
      for (const tool of ['grep', 'awk']) {
        writeFileSync(join(bin, tool), '#!/usr/bin/env bash\nprintf "' + tool + '\\n" >> "$CALL_LOG"\nexec "' + Bun.which(tool)! + '" "$@"\n', { mode: 0o755 });
      }
      try {
        const candidate = [
          "import { beforeAll, afterAll } from 'bun:test';",
          'let engine;',
          'beforeAll(async () => { engine = new PGLiteEngine(); });',
          'afterAll(async () => { await engine.disconnect(); });',
          '',
        ].join('\n');
        const files = Array.from({ length: 200 }, (_, i) => ({ path: 'candidate ' + i + '.test.ts', contents: candidate }));
        const env = { PATH: bin + ':' + process.env.PATH, CALL_LOG: log };
        const r = runLintIn(files, [], env);
        expect(r.status).toBe(0);
        expect(r.stdout).toContain('200 non-serial unit files scanned');
        expect(readFileSync(log, 'utf8').trim().split('\n').length).toBeLessThan(12);
        const bad = runLintIn([...files, { path: 'z bad.test.ts', contents: 'const engine = new PGLiteEngine();\n' }], [], env);
        expect(bad.status).toBe(1);
        expect(bad.stdout).toContain('ERROR: test/z bad.test.ts\n       rule R3:');
        expect(bad.stdout).toContain('         1:const engine = new PGLiteEngine();\nERROR: test/z bad.test.ts\n       rule R4:');
        expect(bad.stdout).toContain('check-test-isolation: FAIL (2 violation(s))');
      } finally {
        rmSync(bin, { recursive: true, force: true });
      }
    });
  });

  describe('R1 — env mutation', () => {
    it('flags process.env.X = assignment', () => {
      const r = runLintIn([
        {
          path: 'env-write.test.ts',
          contents: `process.env.OOPS = 'bad';\n`,
        },
      ]);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('R1');
      expect(r.stdout).toContain('env-write.test.ts');
    });

    it('flags process.env[bracket] = assignment', () => {
      const r = runLintIn([
        {
          path: 'env-bracket.test.ts',
          contents: `process.env['OOPS'] = 'bad';\n`,
        },
      ]);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('R1');
    });

    it('flags delete process.env.X', () => {
      const r = runLintIn([
        {
          path: 'env-delete.test.ts',
          contents: `delete process.env.OOPS;\n`,
        },
      ]);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('R1');
    });

    it('flags Object.assign(process.env, ...)', () => {
      const r = runLintIn([
        {
          path: 'env-assign.test.ts',
          contents: `Object.assign(process.env, { OOPS: 'bad' });\n`,
        },
      ]);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('R1');
    });

    it('flags Reflect.set(process.env, ...)', () => {
      const r = runLintIn([
        {
          path: 'env-reflect.test.ts',
          contents: `Reflect.set(process.env, 'OOPS', 'bad');\n`,
        },
      ]);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('R1');
    });

    it('does NOT flag a comparison like process.env.X === ...', () => {
      const r = runLintIn([
        {
          path: 'env-read.test.ts',
          contents: `if (process.env.X === 'y') {}\n`,
        },
      ]);
      expect(r.status).toBe(0);
    });
  });

  describe('R2 — mock.module()', () => {
    it('flags mock.module(...)', () => {
      const r = runLintIn([
        {
          path: 'mocks.test.ts',
          contents: `import { mock } from 'bun:test';\nmock.module('foo', () => ({}));\n`,
        },
      ]);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('R2');
    });
  });

  describe('R3 — new PGLiteEngine() outside beforeAll context', () => {
    it('flags engine created at module top-level', () => {
      const r = runLintIn([
        {
          path: 'pglite-toplevel.test.ts',
          contents: `import { PGLiteEngine } from '../src/core/pglite-engine.ts';\nconst engine = new PGLiteEngine();\n`,
        },
      ]);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('R3');
    });

    it('does NOT flag engine created within ~50 lines of a beforeAll', () => {
      const r = runLintIn([
        {
          path: 'pglite-ok.test.ts',
          contents:
            `import { beforeAll, afterAll, test, expect } from 'bun:test';\n` +
            `import { PGLiteEngine } from '../src/core/pglite-engine.ts';\n` +
            `let engine: PGLiteEngine;\n` +
            `beforeAll(async () => {\n` +
            `  engine = new PGLiteEngine();\n` +
            `  await engine.connect({});\n` +
            `});\n` +
            `afterAll(async () => { await engine.disconnect(); });\n` +
            `test('x', () => expect(1).toBe(1));\n`,
        },
      ]);
      expect(r.status).toBe(0);
    });

    it('keeps the exact 50-line boundary and reports violations in later candidate files', () => {
      const r = runLintIn([
        { path: 'a clean.test.ts', contents: 'export {};\n' },
        { path: 'b boundary.test.ts', contents: `beforeAll(() => {});\n${'\n'.repeat(49)}new PGLiteEngine();\nafterAll(() => engine.disconnect());\n` },
        { path: 'c outside.test.ts', contents: `beforeAll(() => {});\n${'\n'.repeat(50)}new PGLiteEngine();\nafterAll(() => engine.disconnect());\n` },
      ]);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('rule R3');
      expect(r.stdout).toContain('c outside.test.ts');
      expect(r.stdout).not.toContain('b boundary.test.ts');
      expect(r.stdout).toContain('52:new PGLiteEngine();');
    });
  });

  describe('R4 — afterAll/disconnect pairing', () => {
    it('flags engine creation without afterAll{disconnect}', () => {
      const r = runLintIn([
        {
          path: 'pglite-no-disconnect.test.ts',
          contents:
            `import { beforeAll, test, expect } from 'bun:test';\n` +
            `import { PGLiteEngine } from '../src/core/pglite-engine.ts';\n` +
            `let engine: PGLiteEngine;\n` +
            `beforeAll(async () => { engine = new PGLiteEngine(); await engine.connect({}); });\n` +
            `test('x', () => expect(1).toBe(1));\n`,
        },
      ]);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('R4');
    });
  });

  describe('R5: a configured gateway must be reset', () => {
    const IMPORTS = `import { afterAll, afterEach, beforeAll, test } from 'bun:test';\n` +
      `import { configureGateway, resetGateway } from '../src/core/ai/gateway.ts';\n`;
    const CONFIGURE = `beforeAll(() => configureGateway({ embedding_model: 'litellm:example-embed', env: {} }));\n`;
    const lintOne = (contents: string, path = 'gateway-user.test.ts') => runLintIn([{ path, contents }]);

    it('flags a file that configures the gateway and never resets it, naming the call line', () => {
      const r = lintOne(IMPORTS + CONFIGURE + `test('t', () => {});\n`);
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('test/gateway-user.test.ts');
      expect(r.stdout).toContain('rule R5');
      expect(r.stdout).toContain('3:beforeAll(() => configureGateway(');
    });

    it('control: the same file with an afterEach reset passes', () => {
      const r = lintOne(IMPORTS + CONFIGURE + `afterEach(() => resetGateway());\n`);
      expect(r.status).toBe(0);
      expect(r.stdout).toContain('check-test-isolation: OK');
    });

    it('flags a call through a dynamic import and a call with a space before the paren', () => {
      for (const call of [
        `test('t', async () => { (await import('../src/core/ai/gateway.ts')).configureGateway({ env: {} }); });\n`,
        `beforeAll(() => configureGateway ({ env: {} }));\n`,
      ]) {
        const r = lintOne(call);
        expect(r.status).toBe(1);
        expect(r.stdout).toContain('rule R5');
      }
    });

    it('a reset that only appears in a comment does not count', () => {
      for (const reset of [
        `// afterAll(() => resetGateway()) is handled by a sibling file\n`,
        `/**\n * The suite would call resetGateway() here.\n */\n`,
        `test('t', () => {}); // TODO resetGateway()\n`,
      ]) {
        const r = lintOne(IMPORTS + CONFIGURE + reset);
        expect(r.status).toBe(1);
        expect(r.stdout).toContain('rule R5');
      }
    });

    it('a configureGateway mention that only appears in a comment is not a call', () => {
      for (const mention of [
        `// configureGateway() is set up by the preload\n`,
        `/*\n  configureGateway({ env: {} });\n*/\n`,
        `test('t', () => {}); // unlike configureGateway(), this needs no reset\n`,
      ]) {
        expect(lintOne(mention + `test('u', () => {});\n`).status).toBe(0);
      }
    });

    it('a URL or a glob on a code line does not hide the reset', () => {
      const r = lintOne(
        IMPORTS + CONFIGURE +
        `const fixtures = 'test/*.json';\n` +
        `afterAll(() => { const u = 'http://localhost:4000'; void u; resetGateway(); });\n`,
      );
      expect(r.status).toBe(0);
    });

    it('the subprocess opt-out exempts a file whose call runs in a child script', () => {
      const body = 'const child = `\n  configureGateway({ env: {} });\n`;\nvoid child;\n';
      expect(lintOne(`// isolation-lint: R5-subprocess-only (runs in the spawned child)\n` + body).status).toBe(0);
      const control = lintOne(body);
      expect(control.status).toBe(1);
      expect(control.stdout).toContain('rule R5');
    });

    it('serial files are out of scope for R5', () => {
      expect(lintOne(IMPORTS + CONFIGURE, 'gateway-user.serial.test.ts').status).toBe(0);
    });
  });

  describe('scope', () => {
    it('skips *.serial.test.ts files entirely', () => {
      const r = runLintIn([
        {
          path: 'naughty.serial.test.ts',
          contents: `process.env.OOPS = 'bad';\nimport { mock } from 'bun:test';\nmock.module('foo', () => ({}));\n`,
        },
      ]);
      expect(r.status).toBe(0);
    });

    it('skips test/e2e/ subtree', () => {
      const r = runLintIn([
        {
          path: 'e2e/leak.test.ts',
          contents: `process.env.OOPS = 'bad';\n`,
        },
      ]);
      expect(r.status).toBe(0);
    });
  });

  describe('allowlist', () => {
    it('skips files listed in the allowlist', () => {
      const r = runLintIn(
        [
          {
            path: 'legacy.test.ts',
            contents: `process.env.OOPS = 'bad';\n`,
          },
        ],
        ['test/legacy.test.ts'],
      );
      expect(r.status).toBe(0);
    });

    it('still flags files NOT in the allowlist when allowlist is non-empty', () => {
      const r = runLintIn(
        [
          {
            path: 'allowed.test.ts',
            contents: `process.env.X = 'a';\n`,
          },
          {
            path: 'fresh.test.ts',
            contents: `process.env.Y = 'b';\n`,
          },
        ],
        ['test/allowed.test.ts'],
      );
      expect(r.status).toBe(1);
      expect(r.stdout).toContain('fresh.test.ts');
      expect(r.stdout).not.toContain('allowed.test.ts');
    });

    it('treats # comments and blank lines in allowlist as no-ops', () => {
      const r = runLintIn(
        [
          {
            path: 'legacy.test.ts',
            contents: `process.env.OOPS = 'bad';\n`,
          },
        ],
        ['# legacy file', '', 'test/legacy.test.ts'],
      );
      expect(r.status).toBe(0);
    });
  });
});

describe('check-test-isolation.sh --as-parallel', () => {
  function runAsParallel(files: FakeFile[], targets: string[]): RunResult {
    const dir = mkdtempSync(join(tmpdir(), 'lint-isolation-parallel-'));
    try {
      mkdirSync(join(dir, 'scripts'), { recursive: true });
      writeFileSync(join(dir, 'scripts/check-test-isolation.allowlist'), '');
      for (const f of files) {
        const full = join(dir, 'test', f.path);
        mkdirSync(resolve(full, '..'), { recursive: true });
        writeFileSync(full, f.contents);
      }
      const r = spawnSync('bash', [LINT_SH, '--as-parallel', ...targets.map(t => `test/${t}`)], { cwd: dir, encoding: 'utf-8' });
      return { status: r.status ?? -1, stdout: r.stdout, stderr: r.stderr };
    } finally { rmSync(dir, { recursive: true, force: true }); }
  }

  it('lints a serial file as if it ran in the parallel pool', () => {
    const r = runAsParallel([{ path: 'env.serial.test.ts', contents: "process.env.EXAMPLE_FLAG = '1';\n" }], ['env.serial.test.ts']);
    expect(r.status).toBe(1);
    expect(r.stdout).toContain('ERROR: test/env.serial.test.ts');
    expect(r.stdout).toContain('rule R1');
  });

  it('passes a parallel-safe serial file and lints only the named files', () => {
    const r = runAsParallel([
      { path: 'clean.serial.test.ts', contents: "import { test } from 'bun:test';\ntest('x', () => {});\n" },
      { path: 'other.serial.test.ts', contents: "mock.module('x', () => ({}));\n" },
    ], ['clean.serial.test.ts']);
    expect(r.status).toBe(0);
    expect(r.stdout).toContain('OK (1 ');
  });

  it('requires at least one file', () => {
    const r = runAsParallel([], []);
    expect(r.status).toBe(2);
    expect(r.stderr).toContain('--as-parallel FILE');
  });
});