Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
3.2 KB
Raw Blame
import assert from "node:assert/strict"
import { test } from "node:test"
import { aggregate, normalizeRun, renderMarkdown } from "./flake-report.mjs"

const root = "/repo/evals"

function receipt(entries) {
  return {
    testResults: entries.map((entry) => ({
      name: `${root}/specs/${entry.spec}`,
      status: entry.passed ? "passed" : "failed",
      assertionResults: [{ status: entry.passed ? "passed" : "failed", duration: entry.durationMs }],
    })),
  }
}

test("normalizeRun maps vitest JSON receipts to repo-relative spec results", () => {
  const run = normalizeRun(receipt([{ spec: "a.test.ts", passed: true, durationMs: 120 }]), root)
  assert.deepEqual(run, [{ spec: "specs/a.test.ts", passed: true, skipped: false, durationMs: 120 }])
})

test("normalizeRun rejects non-vitest payloads", () => {
  assert.throws(() => normalizeRun({ nope: true }, root), /vitest JSON reporter payload/)
})

test("normalizeRun marks fully skipped files as skipped", () => {
  const run = normalizeRun(
    {
      testResults: [
        {
          name: `${root}/specs/skipped.test.ts`,
          status: "passed",
          assertionResults: [{ status: "pending" }, { status: "todo" }],
        },
      ],
    },
    root,
  )
  assert.equal(run[0].skipped, true)
})

test("aggregate computes pass rate, flaky, and quarantine flags", () => {
  const runs = [
    normalizeRun(
      receipt([
        { spec: "stable.test.ts", passed: true, durationMs: 100 },
        { spec: "flaky.test.ts", passed: true, durationMs: 200 },
        { spec: "broken.test.ts", passed: false, durationMs: 50 },
      ]),
      root,
    ),
    normalizeRun(
      receipt([
        { spec: "stable.test.ts", passed: true, durationMs: 110 },
        { spec: "flaky.test.ts", passed: false, durationMs: 900 },
        { spec: "broken.test.ts", passed: false, durationMs: 60 },
      ]),
      root,
    ),
  ]
  const result = aggregate(runs, 0.95)
  assert.equal(result.totalRuns, 2)
  assert.equal(result.totalSpecs, 3)
  assert.equal(result.flakySpecs, 1)
  assert.equal(result.quarantineSpecs, 2)

  const flaky = result.specs.find((entry) => entry.spec === "specs/flaky.test.ts")
  assert.equal(flaky.passRate, 0.5)
  assert.equal(flaky.flaky, true)
  assert.equal(flaky.quarantine, true)
  assert.equal(flaky.meanDurationMs, 550)
  assert.equal(flaky.maxDurationMs, 900)

  const broken = result.specs.find((entry) => entry.spec === "specs/broken.test.ts")
  assert.equal(broken.flaky, false)
  assert.equal(broken.quarantine, true)

  const stable = result.specs.find((entry) => entry.spec === "specs/stable.test.ts")
  assert.equal(stable.passRate, 1)
  assert.equal(stable.quarantine, false)

  // Worst pass rate sorts first.
  assert.equal(result.specs[0].passRate <= result.specs[result.specs.length - 1].passRate, true)
})

test("renderMarkdown emits one row per spec with the summary line", () => {
  const runs = [normalizeRun(receipt([{ spec: "a.test.ts", passed: true, durationMs: 10 }]), root)]
  const markdown = renderMarkdown(aggregate(runs, 0.95), "pr")
  assert.match(markdown, /# Flake report — pr lane/)
  assert.match(markdown, /Runs aggregated: 1 · Specs: 1 · Flaky: 0/)
  assert.match(markdown, /\| specs\/a\.test\.ts \| 1 \| 100\.0% \|/)
})