:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
/**
* Playwright reporter half of the test quarantine — enforces `mode: "run"`.
*
* Playwright has no native non-strict xfail, so a `mode: "run"` quarantined
* test still executes (keeping its timing/outcome in the html/junit/json
* reports) and this reporter tolerates a genuine failure: if the ONLY
* unexpected failures in the run are `mode: "run"` quarantined tests, it
* overrides the final run status to `passed` so they don't block CI. A single
* non-quarantined failure keeps the run red.
*
* Scope: this only flips the run-level status the CI exit code reads. The
* per-test outcome in junit-results.xml is written by the separate junit
* reporter and still records the failure, so a per-test consumer of that file
* (e.g. test-health ingestion) sees it as failed. Unlike pytest's xfail, the
* quarantine does not rewrite the per-test result, only the merge gate.
*
* The override is deliberately conservative: it fires ONLY when the run's final
* status is a plain `failed`, no `onError` fired, and every `unexpected` test
* outcome is covered by a `mode: "run"` entry. Anything surfaced through
* `onError` (global setup/teardown throw, uncaught error) or a non-`failed`
* final status (`timedout`, `interrupted`) keeps the run red, because masking a
* run-level problem would be far worse than a flaky test blocking a merge.
*
* Two accepted limitations follow from that conservatism:
* - Reaching `--max-failures` both fires `onError` ("Testing stopped early")
* and finalizes the run as `interrupted`, so once the cap is hit quarantine
* no longer suppresses those failures. That is the safe direction (stay
* red) and the alternative (overriding an `onError` run) is not worth it.
* - A mid-test worker crash (OOM/segfault) is attributed by Playwright to
* that test as `unexpected` rather than through `onError`, so a crash in a
* `mode: "run"` test is tolerated as that test's own failure. Playwright
* restarts the worker and re-runs the rest, so the blast radius is the
* quarantined test itself.
*
* `mode: "skip"` is handled earlier by the auto fixture in
* `utils/playwright-test-core.ts`; skipped tests never reach here as failures.
*
* Fail-open: if the quarantine file is missing or unreadable there are no
* active entries and this reporter is a no-op.
*
* Schema contract + matching: `playwright.quarantine.ts` (and core.py behind it).
*/
import type { FullResult, Reporter, Suite, TestCase } from '@playwright/test/reporter'
import { QuarantineDecision, QuarantineEntry, decideForTest, loadActiveEntries } from './playwright.quarantine'
/**
* Describe titles from the root down, then the test title — the playwright name
* parts. Anonymous `describe(() => {…})` blocks (empty title) are skipped, so
* this matches Playwright's own `titlePath()` that the skip fixture consumes via
* `info.titlePath` — otherwise the two enforcement halves derive different ids.
*/
export function nameParts(test: TestCase): string[] {
const describes: string[] = []
for (let suite: Suite | undefined = test.parent; suite && suite.type === 'describe'; suite = suite.parent) {
if (suite.title) {
describes.unshift(suite.title)
}
}
return [...describes, test.title]
}
export default class QuarantineReporter implements Reporter {
private readonly entries: QuarantineEntry[]
private rootSuite: Suite | undefined
private sawRunError = false
// entries is injectable so the onEnd override can be unit-tested without a
// file. Playwright itself constructs config-registered reporters with a
// reporter-options OBJECT (`{}` when none are configured), so anything that
// is not an array means "load from the quarantine file".
constructor(entries?: QuarantineEntry[] | object) {
this.entries = Array.isArray(entries) ? (entries as QuarantineEntry[]) : loadActiveEntries()
}
// We only emit occasional warnings, so let Playwright keep its default terminal reporter.
printsToStdio(): boolean {
return false
}
onBegin(_config: unknown, suite: Suite): void {
this.rootSuite = suite
}
// Errors outside test execution (worker crash, global setup/teardown, uncaught) land here,
// not as an `unexpected` test — record them so the override below never masks one.
onError(): void {
this.sawRunError = true
}
async onEnd(result: FullResult): Promise<{ status?: FullResult['status'] } | void> {
if (this.entries.length === 0 || !this.rootSuite) {
return
}
// Only a plain 'failed' run is a candidate. 'timedout' (global timeout), 'interrupted',
// and any run-level error are not per-test failures — never override them.
if (result.status !== 'failed' || this.sawRunError) {
return
}
const tolerated: QuarantineDecision[] = []
let realFailures = 0
for (const test of this.rootSuite.allTests()) {
if (test.outcome() !== 'unexpected') {
continue // Passed, skipped, or recovered-on-retry (flaky) — not a blocking failure.
}
const decision = decideForTest(this.entries, test.location.file, nameParts(test))
if (decision?.mode === 'run') {
tolerated.push(decision)
} else {
realFailures += 1
}
}
if (tolerated.length === 0) {
return // Nothing quarantined failed; the run stays red on its own merits.
}
for (const decision of tolerated) {
// eslint-disable-next-line no-console
console.warn(`[quarantine] tolerated failure in ${decision.label}`)
}
if (realFailures > 0) {
return // Real failures remain — keep the run red.
}
// eslint-disable-next-line no-console
console.warn(
`[quarantine] overriding run status to passed — the only ${tolerated.length} unexpected ` +
`failure(s) are quarantined (mode: run)`
)
return { status: 'passed' }
}
}