Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.8 KB
Raw Blame
// coverage-baseline-gate.test.ts — behavioral, fixture-locked tests for
// scripts/coverage-baseline-gate.ts, plus unit coverage of the pure section
// builder in scripts/update-coverage-baseline.ts.
//
// The git-show source is injected via the documented test seam
// COVERAGE_BASELINE_JSON_OVERRIDE (literal JSON; sentinels ABSENT_ON_MASTER
// and GIT_FAILURE), so tests never depend on the real origin/master.

import { afterAll, beforeAll, describe, expect, it } from "bun:test";
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { compareCorpus } from "../../scripts/coverage-baseline-gate.ts";
import { buildCorpusSection } from "../../scripts/update-coverage-baseline.ts";

const REPO_ROOT = join(import.meta.dir, "..", "..");

let tmp: string;
beforeAll(() => {
  tmp = mkdtempSync(join(tmpdir(), "coverage-baseline-gate-test-"));
});
afterAll(() => {
  rmSync(tmp, { recursive: true, force: true });
});

let fixtureSeq = 0;
function writeSummary(
  total: { pct: number },
  dirs: Record<string, { pct: number }>,
  degraded = false,
  neverLoadedCount?: number,
): string {
  const p = join(tmp, `summary-${fixtureSeq++}.json`);
  const expand = (x: { pct: number }) => ({ lines: 1000, covered: Math.round(x.pct * 10), pct: x.pct });
  writeFileSync(
    p,
    JSON.stringify({
      corpus: "prCorpus",
      degraded,
      total: expand(total),
      dirs: Object.fromEntries(Object.entries(dirs).map(([k, v]) => [k, expand(v)])),
      ...(neverLoadedCount === undefined ? {} : { neverLoaded: { count: neverLoadedCount, files: [] } }),
    }) + "\n",
    "utf8",
  );
  return p;
}

const BASELINE = {
  provisional: false,
  prCorpus: {
    global: { lines: 1000, covered: 800, pct: 80 },
    dirs: { "src/core": { lines: 500, covered: 350, pct: 70 } },
    files: {},
  },
  fullCorpus: null,
  watchlist: [],
};

interface RunResult {
  code: number;
  stdout: string;
  stderr: string;
}

function runGate(opts: {
  summary: string;
  corpus?: string;
  baseline?: string;
  enforce?: string;
  worktreePath?: string;
}): RunResult {
  const res = Bun.spawnSync(
    [
      "bun",
      join(REPO_ROOT, "scripts", "coverage-baseline-gate.ts"),
      "--summary",
      opts.summary,
      "--corpus",
      opts.corpus ?? "prCorpus",
    ],
    {
      cwd: REPO_ROOT,
      env: {
        ...process.env,
        COVERAGE_BASELINE_JSON_OVERRIDE: opts.baseline ?? JSON.stringify(BASELINE),
        COVERAGE_GATE_ENFORCE: opts.enforce ?? "0",
        ...(opts.worktreePath ? { COVERAGE_BASELINE_WORKTREE_PATH: opts.worktreePath } : {}),
      },
      stdout: "pipe",
      stderr: "pipe",
    },
  );
  return { code: res.exitCode, stdout: res.stdout.toString(), stderr: res.stderr.toString() };
}

describe("regression thresholds", () => {
  it("within thresholds → PASS (report-only WOULD PASS; enforcing PASS)", () => {
    const summary = writeSummary({ pct: 79.6 }, { "src/core": { pct: 69.1 } });
    const reportOnly = runGate({ summary });
    expect(reportOnly.code).toBe(0);
    expect(reportOnly.stdout).toContain("WOULD PASS");

    const enforcing = runGate({ summary, enforce: "1" });
    expect(enforcing.code).toBe(0);
    expect(enforcing.stdout).toContain("PASS: no coverage regression");
  });

  it("global drop of exactly 0.5pp is NOT a regression (strictly-greater rule)", () => {
    const summary = writeSummary({ pct: 79.5 }, { "src/core": { pct: 70 } });
    const res = runGate({ summary, enforce: "1" });
    expect(res.code).toBe(0);
  });

  it("global regression > 0.5pp → WOULD FAIL report-only (exit 0), FAIL enforcing (exit 1)", () => {
    const summary = writeSummary({ pct: 79.0 }, { "src/core": { pct: 70 } });
    const reportOnly = runGate({ summary });
    expect(reportOnly.code).toBe(0);
    expect(reportOnly.stdout).toContain("WOULD FAIL");
    expect(reportOnly.stdout).toContain("REGRESSION: global");

    const enforcing = runGate({ summary, enforce: "1" });
    expect(enforcing.code).toBe(1);
    expect(enforcing.stdout).toContain("FAIL: coverage regression");
  });

  it("per-dir regression > 1.0pp fails even when global holds", () => {
    const summary = writeSummary({ pct: 80 }, { "src/core": { pct: 68.9 } });
    const res = runGate({ summary, enforce: "1" });
    expect(res.code).toBe(1);
    expect(res.stdout).toContain("REGRESSION: src/core");
  });

  it("per-dir drop of exactly 1.0pp passes", () => {
    const summary = writeSummary({ pct: 80 }, { "src/core": { pct: 69 } });
    const res = runGate({ summary, enforce: "1" });
    expect(res.code).toBe(0);
  });
});

describe("never-loaded regression (shrinking-denominator defense)", () => {
  const BASELINE_WITH_NL = {
    ...BASELINE,
    prCorpus: { ...BASELINE.prCorpus, neverLoadedCount: 5 },
  };

  it("run count EXCEEDS the baseline → WOULD FAIL report-only, FAIL enforcing (exit 1)", () => {
    // pcts hold steady — only the never-loaded count regresses (tests were
    // deleted, shrinking the denominator).
    const summary = writeSummary({ pct: 80 }, { "src/core": { pct: 70 } }, false, 7);
    const reportOnly = runGate({ summary, baseline: JSON.stringify(BASELINE_WITH_NL) });
    expect(reportOnly.code).toBe(0);
    expect(reportOnly.stdout).toContain("WOULD FAIL");
    expect(reportOnly.stdout).toContain("REGRESSION: neverLoaded: 5 → 7");

    const enforcing = runGate({ summary, baseline: JSON.stringify(BASELINE_WITH_NL), enforce: "1" });
    expect(enforcing.code).toBe(1);
    expect(enforcing.stdout).toContain("FAIL: coverage regression");
  });

  it("run count EQUAL to the baseline passes", () => {
    const summary = writeSummary({ pct: 80 }, { "src/core": { pct: 70 } }, false, 5);
    const res = runGate({ summary, baseline: JSON.stringify(BASELINE_WITH_NL), enforce: "1" });
    expect(res.code).toBe(0);
    expect(res.stdout).toContain("PASS: no coverage regression");
  });

  it("null/absent baseline neverLoadedCount skips the check entirely", () => {
    // BASELINE has no neverLoadedCount field: even a huge run count passes.
    const absentField = writeSummary({ pct: 80 }, { "src/core": { pct: 70 } }, false, 999);
    const resAbsent = runGate({ summary: absentField, enforce: "1" });
    expect(resAbsent.code).toBe(0);

    // Explicit null skips too.
    const nullBaseline = { ...BASELINE, prCorpus: { ...BASELINE.prCorpus, neverLoadedCount: null } };
    const resNull = runGate({ summary: absentField, baseline: JSON.stringify(nullBaseline), enforce: "1" });
    expect(resNull.code).toBe(0);

    // And a summary WITHOUT neverLoaded is skipped even when the baseline
    // carries a count (older summary shapes stay comparable).
    const noSummaryField = writeSummary({ pct: 80 }, { "src/core": { pct: 70 } });
    const resNoSummary = runGate({ summary: noSummaryField, baseline: JSON.stringify(BASELINE_WITH_NL), enforce: "1" });
    expect(resNoSummary.code).toBe(0);
  });
});

describe("report-only downgrades", () => {
  it("provisional baseline → report-only even with a regression under enforcement", () => {
    const summary = writeSummary({ pct: 70 }, { "src/core": { pct: 60 } });
    const res = runGate({
      summary,
      enforce: "1",
      baseline: JSON.stringify({ ...BASELINE, provisional: true }),
    });
    expect(res.code).toBe(0);
    expect(res.stdout).toContain("REPORT-ONLY (baseline is provisional)");
    expect(res.stdout).toContain("WOULD FAIL");
  });

  it("degraded summary → report-only even with a regression under enforcement", () => {
    const summary = writeSummary({ pct: 70 }, { "src/core": { pct: 60 } }, true);
    const res = runGate({ summary, enforce: "1" });
    expect(res.code).toBe(0);
    expect(res.stdout).toContain("DEGRADED");
    expect(res.stdout).toContain("WOULD FAIL");
  });
});

describe("ungated first landings", () => {
  it("null corpus section → ungated, exit 0", () => {
    const summary = writeSummary({ pct: 10 }, {});
    const res = runGate({ summary, corpus: "fullCorpus", enforce: "1" });
    expect(res.code).toBe(0);
    expect(res.stdout).toContain("ungated first landing");
  });

  it("baseline absent on master → ungated, exit 0", () => {
    const summary = writeSummary({ pct: 10 }, {});
    const res = runGate({ summary, enforce: "1", baseline: "ABSENT_ON_MASTER" });
    expect(res.code).toBe(0);
    expect(res.stdout).toContain("ungated first landing");
  });
});

describe("infrastructure errors", () => {
  it("unresolvable ref → exit 2", () => {
    const summary = writeSummary({ pct: 80 }, {});
    const res = runGate({ summary, baseline: "GIT_FAILURE" });
    expect(res.code).toBe(2);
    expect(res.stderr).toContain("infrastructure error");
  });

  it("deletion defense: baseline on master but working-tree copy missing → exit 2", () => {
    const summary = writeSummary({ pct: 80 }, { "src/core": { pct: 70 } });
    const res = runGate({ summary, worktreePath: join(tmp, "deleted-baseline.json") });
    expect(res.code).toBe(2);
    expect(res.stderr).toContain("working-tree copy is missing");
  });

  it("missing summary → exit 2; bad --corpus → exit 2", () => {
    const missing = runGate({ summary: join(tmp, "nope.json") });
    expect(missing.code).toBe(2);
    const badCorpus = runGate({ summary: writeSummary({ pct: 80 }, {}), corpus: "weekendCorpus" });
    expect(badCorpus.code).toBe(2);
  });
});

describe("compareCorpus (unit)", () => {
  it("ignores dirs absent from the summary and reports both regression kinds", () => {
    const report = compareCorpus(
      {
        global: { lines: 100, covered: 80, pct: 80 },
        dirs: {
          "src/core": { lines: 50, covered: 35, pct: 70 },
          "src/gone": { lines: 10, covered: 9, pct: 90 },
        },
        files: {},
      },
      { pct: 79 },
      { "src/core": { pct: 68 } },
    );
    expect(report.pass).toBe(false);
    expect(report.regressions.length).toBe(2); // global + src/core; src/gone skipped
  });
});

describe("buildCorpusSection (unit, update-coverage-baseline.ts)", () => {
  it("limits files to watchlist entries present in the summary", () => {
    const section = buildCorpusSection(
      {
        total: { lines: 10, covered: 5, pct: 50 },
        dirs: { src: { lines: 10, covered: 5, pct: 50 } },
        files: {
          "src/cli.ts": { lines: 5, covered: 1, pct: 20 },
          "src/unwatched.ts": { lines: 5, covered: 4, pct: 80 },
        },
      },
      ["src/cli.ts", "src/core/migrate.ts"],
    );
    expect(section.global.pct).toBe(50);
    expect(Object.keys(section.files)).toEqual(["src/cli.ts"]);
    expect(section.neverLoadedCount).toBe(null); // summary lacks neverLoaded
  });

  it("records the summary's neverLoaded.count so the gate can catch shrinking denominators", () => {
    const section = buildCorpusSection(
      {
        total: { lines: 10, covered: 5, pct: 50 },
        dirs: {},
        files: {},
        neverLoaded: { count: 42 },
      },
      [],
    );
    expect(section.neverLoadedCount).toBe(42);
  });
});