Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add a unified, dry-run ci-conductor quarantine gate (#76752)

Nemanja Glumac7098c5576a56
diff --git a/.github/actions/quarantine-gate/action.yml b/.github/actions/quarantine-gate/action.yml
new file mode 100644
index 00000000000..7abf517a78b
--- /dev/null
+++ b/.github/actions/quarantine-gate/action.yml
@@ -0,0 +1,46 @@
+name: CI Conductor quarantine gate (dry run)
+description: >-
+ Check this job's test failures against ci-conductor's quarantine list and print
+ a verdict. DRY RUN — it never fails the job. No-ops unless CI_CONDUCTOR_BASE_URL
+ is set.
+
+inputs:
+ adapter:
+ description: Which gate to run — one of `backend`, `frontend`, `e2e`.
+ required: true
+ test-suite:
+ description: >-
+ Suite label to gate (CI_CONDUCTOR_TEST_SUITE). MUST match the label this
+ job reports under, or the gate checks the wrong quarantine list.
+ required: true
+
+runs:
+ using: composite
+ steps:
+ - name: Detect bun
+ id: bun
+ shell: bash
+ run: |
+ if command -v bun >/dev/null; then
+ echo "present=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "present=false" >> "$GITHUB_OUTPUT"
+ fi
+ - name: Install bun
+ if: steps.bun.outputs.present != 'true'
+ uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+ with:
+ bun-version: "1.3.14"
+ token: ${{ github.token }}
+ - name: Check failures against the quarantine list
+ shell: bash
+ env:
+ CI_CONDUCTOR_TEST_SUITE: ${{ inputs.test-suite }}
+ ADAPTER: ${{ inputs.adapter }}
+ QUARANTINE_DRY_RUN: "true"
+ run: |
+ case "$ADAPTER" in
+ backend|frontend|e2e) ;;
+ *) echo "::error::quarantine-gate: unknown adapter '$ADAPTER' (expected backend|frontend|e2e)"; exit 1 ;;
+ esac
+ bun "release/ci-conductor/src/check-${ADAPTER}-quarantine.ts"
diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.yml
index d36bff76e41..9940e6debab 100644
--- a/.github/actions/test-driver/action.yml
+++ b/.github/actions/test-driver/action.yml
@@ -61,6 +61,13 @@ runs:
with:
test-suite: ${{ inputs.junit-name }}
adapter: backend
+ - name: Quarantine gate (dry run)
+ if: ${{ always() && steps.run-test.outcome != 'success' }}
+ continue-on-error: true
+ uses: ./.github/actions/quarantine-gate
+ with:
+ adapter: backend
+ test-suite: ${{ inputs.junit-name }}
- name: Publish Test Report (JUnit)
uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
# As per the documentation: https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#steps-context
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index ff156abc451..09a7f2a7314 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -214,6 +214,15 @@ jobs:
test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}
adapter: backend
+ - name: Quarantine gate (dry run)
+ if: ${{ always() && steps.run-java-tests.outcome != 'success' }}
+ continue-on-error: true
+ timeout-minutes: 3
+ uses: ./.github/actions/quarantine-gate
+ with:
+ adapter: backend
+ test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}
+
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always() && (steps.run-java-tests.conclusion != 'skipped' || steps.run-java-tests-21.conclusion != 'skipped')
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index e741d7d1b1b..7726a91c23d 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -214,20 +214,16 @@ jobs:
trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
previous-step-outcome: ${{ inputs.specs && steps.tagged-e2e.outcome || steps.split-e2e.outcome }}
- # Quarantine gate (DEV-2082). Compare this job's ultimate test failures
- # (recorded by after:spec into ./target/quarantine-failures.jsonl) against
- # ci-conductor's quarantine list and decide whether the job should pass:
- # pass if every failure is quarantined, fail if any isn't. This mirrors
- # what Trunk's analytics-uploader does for us today. Runs in dry-run mode
- # for now (QUARANTINE_DRY_RUN defaults to true) — it only prints the
- # verdict and never fails the job. `continue-on-error` is belt-and-braces
- # while it's observational; drop both it and the dry run to enforce.
- name: Quarantine gate (dry run)
+ # always(), unlike backend/frontend: the run step is continue-on-error
+ # (Trunk governs its outcome), so its result can't gate on real failures.
if: always()
continue-on-error: true
- timeout-minutes: 2
- shell: bash
- run: bun e2e/support/check-quarantine.ts
+ timeout-minutes: 3
+ uses: ./.github/actions/quarantine-gate
+ with:
+ adapter: e2e
+ test-suite: e2e
- name: Upload Cypress Artifacts upon failure
uses: actions/upload-artifact@v7
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
index 15bfcbc00b0..23bd979fb4b 100644
--- a/.github/workflows/frontend.yml
+++ b/.github/workflows/frontend.yml
@@ -125,6 +125,14 @@ jobs:
with:
test-suite: fe-tests-unit
adapter: frontend
+ - name: Quarantine gate (dry run)
+ if: ${{ always() && steps.fe-tests.outcome != 'success' }}
+ continue-on-error: true
+ timeout-minutes: 3
+ uses: ./.github/actions/quarantine-gate
+ with:
+ adapter: frontend
+ test-suite: fe-tests-unit
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always() && github.event_name != 'workflow_dispatch'
diff --git a/e2e/support/check-quarantine.ts b/e2e/support/check-quarantine.ts
deleted file mode 100644
index 256d482a10e..00000000000
--- a/e2e/support/check-quarantine.ts
+++ /dev/null
@@ -1,167 +0,0 @@
-#!/usr/bin/env bun
-
-// Quarantine gate (DEV-2082). After an e2e job runs, compare the tests that
-// *ultimately failed* against the quarantine list served by ci-conductor and
-// decide whether the job should pass or fail:
-//
-// - No failures -> pass (nothing to gate)
-// - Every failure is quarantined -> pass (exit cleanly)
-// - Any failure is NOT quarantined -> fail
-//
-// This mirrors what Trunk's analytics-uploader currently does for us; the goal
-// is to eventually replace that. For now the gate runs in DRY-RUN mode: it
-// computes and prints the verdict but always exits 0, so it can be observed in
-// CI without affecting job outcomes. Flip QUARANTINE_DRY_RUN=false to enforce.
-//
-// The failures come from the file written by after:spec
-// (`recordFailedTestsForQuarantine` in ci_conductor.ts). Both that file and
-// ci-conductor's quarantine list derive {test_name, test_path, file_path} from
-// the same Cypress title array, so the comparison here is exact.
-
-import { readFileSync } from "node:fs";
-
-import {
- type FailedTest,
- type QuarantineEntry,
- compareFailedToQuarantine,
-} from "./quarantine-compare";
-
-const {
- CI_CONDUCTOR_BASE_URL,
- CI_CONDUCTOR_WEBHOOK_SECRET,
- // Default to a dry run: compute and print the verdict, but never fail the
- // job. Set to "false" to actually gate the build on the verdict.
- QUARANTINE_DRY_RUN,
- QUARANTINE_FAILURES_FILE,
-} = process.env;
-
-const isDryRun = QUARANTINE_DRY_RUN !== "false";
-
-// The test suite to gate. ci-conductor keys its quarantine list by suite.
-const TEST_SUITE = "e2e";
-
-const failuresFile =
- QUARANTINE_FAILURES_FILE ?? "./target/quarantine-failures.jsonl";
-
-/** Read the run's failed tests from the JSONL file. */
-function readFailedTests(file: string): FailedTest[] {
- let raw: string;
- try {
- raw = readFileSync(file, "utf8");
- } catch {
- console.log(`[quarantine] no failures file at ${file}; nothing to gate.`);
- return [];
- }
-
- return raw
- .split("\n")
- .map((line) => line.trim())
- .filter((line) => line !== "")
- .map((line): FailedTest | null => {
- try {
- return JSON.parse(line) as FailedTest;
- } catch {
- console.error(`[quarantine] skipping unparseable line: ${line}`);
- return null;
- }
- })
- .filter((test): test is FailedTest => test !== null);
-}
-
-/**
- * Build the quarantine list URL from the ci-conductor base origin secret. The
- * reporter posts to ".../webhooks/failed-tests"; the quarantine list lives at
- * ".../api/quarantine" on the same host.
- */
-function quarantineUrl(): string | null {
- if (!CI_CONDUCTOR_BASE_URL) {
- return null;
- }
- const base = CI_CONDUCTOR_BASE_URL.replace(/\/+$/, "");
- return `${base}/api/quarantine?suite=${TEST_SUITE}`;
-}
-
-/** Fetch the quarantine list, or null if it can't be retrieved. */
-async function fetchQuarantine(): Promise<QuarantineEntry[] | null> {
- const url = quarantineUrl();
- if (!url) {
- console.log(
- "[quarantine] CI_CONDUCTOR_BASE_URL is unset; cannot fetch the quarantine list.",
- );
- return null;
- }
- try {
- const headers: Record<string, string> = {};
- if (CI_CONDUCTOR_WEBHOOK_SECRET) {
- headers["x-internal-secret"] = CI_CONDUCTOR_WEBHOOK_SECRET;
- }
- const response = await fetch(url, { headers });
- if (!response.ok) {
- console.error(
- `[quarantine] GET ${url} returned ${response.status} ${response.statusText}`,
- );
- return null;
- }
- const body = (await response.json()) as { tests?: QuarantineEntry[] };
- return body.tests ?? [];
- } catch (error) {
- console.error(`[quarantine] failed to fetch ${url}`, error);
- return null;
- }
-}
-
-/** Print the verdict and, outside dry run, set the exit code accordingly. */
-function finish(shouldFail: boolean, reason: string): void {
- const verdict = shouldFail ? "FAIL" : "PASS";
- const mode = isDryRun ? " (dry run — not enforced)" : "";
- console.log(`[quarantine] verdict: ${verdict}${mode} — ${reason}.`);
- if (shouldFail && !isDryRun) {
- process.exitCode = 1;
- }
-}
-
-async function main(): Promise<void> {
- const failed = readFailedTests(failuresFile);
-
- if (failed.length === 0) {
- console.log("[quarantine] no failed tests; passing.");
- return;
- }
-
- const quarantine = await fetchQuarantine();
- if (quarantine === null) {
- // We couldn't read the list, so we can't confirm everything is
- // quarantined — that's a failing verdict (enforced only outside dry run).
- finish(true, "could not fetch the quarantine list");
- return;
- }
-
- const { quarantined, unquarantined } = compareFailedToQuarantine(
- failed,
- quarantine,
- );
-
- console.log(
- `[quarantine] ${failed.length} failed test(s); ${quarantine.length} test(s) in the ${TEST_SUITE} quarantine list.`,
- );
- const describe = (test: FailedTest) =>
- `${test.test_name} (${test.file_path ?? "unknown file"})`;
- quarantined.forEach((test) =>
- console.log(` 🔒 quarantined: ${describe(test)}`),
- );
- unquarantined.forEach((test) =>
- console.log(` 🚨 NOT quarantined: ${describe(test)}`),
- );
-
- finish(
- unquarantined.length > 0,
- unquarantined.length > 0
- ? `${unquarantined.length} failed test(s) are not quarantined`
- : "every failed test is quarantined",
- );
-}
-
-// Only run when invoked directly (`bun check-quarantine.ts`), not on import.
-if (import.meta.main) {
- await main();
-}
diff --git a/e2e/support/ci_conductor.ts b/e2e/support/ci_conductor.ts
index 172e315aa64..60c9dff8a31 100644
--- a/e2e/support/ci_conductor.ts
+++ b/e2e/support/ci_conductor.ts
@@ -267,7 +267,8 @@ export function extractFailedTests(
* can read the whole job's failures from a single file. Override with
* QUARANTINE_FAILURES_FILE. Note these are the SAME fields ci-conductor stores
* in its quarantine list (both derived from the same Cypress title array), so
- * the gate can compare them exactly. See `check-quarantine.ts`.
+ * the gate can compare them exactly. See
+ * `release/ci-conductor/src/check-e2e-quarantine.ts`.
*/
const QUARANTINE_FAILURES_FILE =
process.env.QUARANTINE_FAILURES_FILE ?? "./target/quarantine-failures.jsonl";
diff --git a/e2e/support/quarantine-compare.ts b/e2e/support/quarantine-compare.ts
deleted file mode 100644
index 13e227a10b5..00000000000
--- a/e2e/support/quarantine-compare.ts
+++ /dev/null
@@ -1,59 +0,0 @@
-// Pure comparison logic for the quarantine gate (DEV-2082), split out from the
-// `check-quarantine.ts` executable so it can be unit tested without the script's
-// CLI side effects (`import.meta.main`, env reads, `fetch`).
-
-import _ from "underscore";
-
-/** One quarantined test as served by ci-conductor's `/api/quarantine`. */
-export type QuarantineEntry = {
- test_name: string;
- test_suite: string;
- test_path: string;
- file_path: string;
-};
-
-/** A test that ultimately failed in this run, as recorded by after:spec. */
-export type FailedTest = {
- test_name: string;
- test_path: string | null;
- file_path: string | null;
-};
-
-/**
- * Identity key for a test: the spec file path, describe path, and leaf test
- * name as a JSON tuple. ci-conductor's quarantine list and our after:spec
- * recorder both derive these three fields from the same Cypress title array
- * (file_path = `spec.relative`, test_path = the joined `describe` titles), so
- * they match exactly. JSON-encoding keeps the parts distinct, so tuples that
- * differ only in where a boundary falls can't collide.
- */
-export function matchKey(
- filePath: string | null | undefined,
- testPath: string | null | undefined,
- testName: string,
-): string {
- return JSON.stringify([filePath ?? "", testPath ?? "", testName]);
-}
-
-/**
- * Partition the run's failed tests into those that are quarantined and those
- * that are not, matching on exact {file_path, test_path, test_name} identity.
- * A single pass over the failures, so the gate doesn't repeat the (potentially
- * large) comparison when it later logs or counts each bucket.
- */
-export function compareFailedToQuarantine(
- failedTests: FailedTest[],
- quarantineEntries: QuarantineEntry[],
-): { quarantined: FailedTest[]; unquarantined: FailedTest[] } {
- const quarantinedKeys = new Set(
- quarantineEntries.map((q) =>
- matchKey(q.file_path, q.test_path, q.test_name),
- ),
- );
- const [quarantined, unquarantined] = _.partition(failedTests, (test) =>
- quarantinedKeys.has(
- matchKey(test.file_path, test.test_path, test.test_name),
- ),
- );
- return { quarantined, unquarantined };
-}
diff --git a/e2e/support/quarantine-compare.unit.spec.ts b/e2e/support/quarantine-compare.unit.spec.ts
deleted file mode 100644
index a5ec8406923..00000000000
--- a/e2e/support/quarantine-compare.unit.spec.ts
+++ /dev/null
@@ -1,114 +0,0 @@
-import {
- type FailedTest,
- type QuarantineEntry,
- compareFailedToQuarantine,
- matchKey,
-} from "./quarantine-compare";
-
-const failed = (
- test_name: string,
- test_path: string | null = "Suite",
- file_path: string | null = "e2e/test/foo.cy.spec.ts",
-): FailedTest => ({ test_name, test_path, file_path });
-
-const quarantined = (
- test_name: string,
- test_path = "Suite",
- file_path = "e2e/test/foo.cy.spec.ts",
-): QuarantineEntry => ({
- test_name,
- test_path,
- file_path,
- test_suite: "e2e",
-});
-
-describe("compareFailedToQuarantine", () => {
- it("puts every failure in `unquarantined` when nothing is quarantined", () => {
- const failures = [failed("a"), failed("b")];
-
- const { quarantined: q, unquarantined } = compareFailedToQuarantine(
- failures,
- [],
- );
-
- expect(q).toEqual([]);
- expect(unquarantined).toEqual(failures);
- });
-
- it("puts every failure in `quarantined` when all are listed", () => {
- const failures = [failed("a"), failed("b")];
-
- const { quarantined: q, unquarantined } = compareFailedToQuarantine(
- failures,
- [quarantined("a"), quarantined("b")],
- );
-
- expect(q).toEqual(failures);
- expect(unquarantined).toEqual([]);
- });
-
- it("partitions a mixed set", () => {
- const a = failed("a");
- const b = failed("b");
- const c = failed("c");
-
- const { quarantined: q, unquarantined } = compareFailedToQuarantine(
- [a, b, c],
- [quarantined("a"), quarantined("c")],
- );
-
- expect(q).toEqual([a, c]);
- expect(unquarantined).toEqual([b]);
- });
-
- it("returns two empty buckets for no failures", () => {
- expect(compareFailedToQuarantine([], [quarantined("a")])).toEqual({
- quarantined: [],
- unquarantined: [],
- });
- });
-
- it("matches on all three fields — same name, different file is not a match", () => {
- const failure = failed("a", "Suite", "e2e/test/foo.cy.spec.ts");
-
- const { unquarantined } = compareFailedToQuarantine(
- [failure],
- [quarantined("a", "Suite", "e2e/test/bar.cy.spec.ts")],
- );
-
- expect(unquarantined).toEqual([failure]);
- });
-
- it("matches on all three fields — same name/file, different describe path is not a match", () => {
- const failure = failed("a", "Suite > inner");
-
- const { unquarantined } = compareFailedToQuarantine(
- [failure],
- [quarantined("a", "Suite > other")],
- );
-
- expect(unquarantined).toEqual([failure]);
- });
-
- it("treats a null path on the failure as an empty string for matching", () => {
- const failure = failed("a", null, null);
-
- const { quarantined: q } = compareFailedToQuarantine(
- [failure],
- [quarantined("a", "", "")],
- );
-
- expect(q).toEqual([failure]);
- });
-});
-
-describe("matchKey", () => {
- it("normalizes null/undefined paths to empty strings", () => {
- expect(matchKey(null, undefined, "a")).toBe(matchKey("", "", "a"));
- });
-
- it("does not collide when a boundary between fields shifts", () => {
- // ["a", "b", "c"] vs ["ab", "", "c"] must stay distinct.
- expect(matchKey("a", "b", "c")).not.toBe(matchKey("ab", "", "c"));
- });
-});
diff --git a/release/ci-conductor/src/check-backend-quarantine.ts b/release/ci-conductor/src/check-backend-quarantine.ts
new file mode 100644
index 00000000000..f1828e00867
--- /dev/null
+++ b/release/ci-conductor/src/check-backend-quarantine.ts
@@ -0,0 +1,31 @@
+#!/usr/bin/env bun
+
+import { normalizeBackendJunit } from "./adapters/backend.ts";
+import {
+ applyQuarantineGate,
+ junitFailuresToFailedTests,
+} from "./quarantine.ts";
+import { log } from "./util.ts";
+
+async function main(): Promise<void> {
+ log("backend quarantine gate starting");
+ const env = process.env;
+ const suite =
+ env.CI_CONDUCTOR_TEST_SUITE ||
+ (env.DRIVERS ? `driver-${env.DRIVERS}` : "backend");
+ await applyQuarantineGate({
+ suite,
+ failures: junitFailuresToFailedTests(normalizeBackendJunit()),
+ baseUrl: env.CI_CONDUCTOR_BASE_URL,
+ secret: env.CI_CONDUCTOR_WEBHOOK_SECRET,
+ dryRun: env.QUARANTINE_DRY_RUN !== "false",
+ });
+}
+
+main().catch((error) => {
+ // Last line of defense: the gate must never throw into the job.
+ console.error(
+ "[ci-conductor] backend quarantine gate failed (ignored)",
+ error,
+ );
+});
diff --git a/release/ci-conductor/src/check-e2e-quarantine.ts b/release/ci-conductor/src/check-e2e-quarantine.ts
new file mode 100644
index 00000000000..38634879d89
--- /dev/null
+++ b/release/ci-conductor/src/check-e2e-quarantine.ts
@@ -0,0 +1,66 @@
+#!/usr/bin/env bun
+
+import { readFileSync } from "node:fs";
+
+import { type FailedTest, applyQuarantineGate } from "./quarantine.ts";
+import { log } from "./util.ts";
+
+const {
+ CI_CONDUCTOR_BASE_URL,
+ CI_CONDUCTOR_WEBHOOK_SECRET,
+ CI_CONDUCTOR_TEST_SUITE,
+ // Default to a dry run: compute and print the verdict, but never fail the job.
+ QUARANTINE_DRY_RUN,
+ QUARANTINE_FAILURES_FILE,
+} = process.env;
+
+const isDryRun = QUARANTINE_DRY_RUN !== "false";
+
+// ci-conductor keys its quarantine list by suite; e2e reports under "e2e".
+const TEST_SUITE = CI_CONDUCTOR_TEST_SUITE || "e2e";
+
+const failuresFile =
+ QUARANTINE_FAILURES_FILE ?? "./target/quarantine-failures.jsonl";
+
+/** Read the run's ultimate failures from the JSONL file after:spec appended. */
+function readFailedTests(file: string): FailedTest[] {
+ let raw: string;
+ try {
+ raw = readFileSync(file, "utf8");
+ } catch {
+ log(`no failures file at ${file}; nothing to gate.`);
+ return [];
+ }
+
+ return raw
+ .split("\n")
+ .map((line) => line.trim())
+ .filter((line) => line !== "")
+ .map((line): FailedTest | null => {
+ try {
+ return JSON.parse(line) as FailedTest;
+ } catch {
+ log(`skipping unparseable line: ${line}`);
+ return null;
+ }
+ })
+ .filter((test): test is FailedTest => test !== null);
+}
+
+async function main(): Promise<void> {
+ await applyQuarantineGate({
+ suite: TEST_SUITE,
+ failures: readFailedTests(failuresFile),
+ baseUrl: CI_CONDUCTOR_BASE_URL,
+ secret: CI_CONDUCTOR_WEBHOOK_SECRET,
+ dryRun: isDryRun,
+ });
+}
+
+// Only run when invoked directly (`bun src/check-e2e-quarantine.ts`), not on import.
+if (import.meta.main) {
+ main().catch((error) => {
+ // Last line of defense: the gate must never throw into the job.
+ console.error("[ci-conductor] e2e quarantine gate failed (ignored)", error);
+ });
+}
diff --git a/release/ci-conductor/src/check-frontend-quarantine.ts b/release/ci-conductor/src/check-frontend-quarantine.ts
new file mode 100644
index 00000000000..60fa4dde5c7
--- /dev/null
+++ b/release/ci-conductor/src/check-frontend-quarantine.ts
@@ -0,0 +1,29 @@
+#!/usr/bin/env bun
+
+import { normalizeFrontendJunit } from "./adapters/frontend.ts";
+import {
+ applyQuarantineGate,
+ junitFailuresToFailedTests,
+} from "./quarantine.ts";
+import { log } from "./util.ts";
+
+async function main(): Promise<void> {
+ log("frontend quarantine gate starting");
+ const env = process.env;
+ const suite = env.CI_CONDUCTOR_TEST_SUITE || "frontend";
+ await applyQuarantineGate({
+ suite,
+ failures: junitFailuresToFailedTests(normalizeFrontendJunit()),
+ baseUrl: env.CI_CONDUCTOR_BASE_URL,
+ secret: env.CI_CONDUCTOR_WEBHOOK_SECRET,
+ dryRun: env.QUARANTINE_DRY_RUN !== "false",
+ });
+}
+
+main().catch((error) => {
+ // Last line of defense: the gate must never throw into the job.
+ console.error(
+ "[ci-conductor] frontend quarantine gate failed (ignored)",
+ error,
+ );
+});
diff --git a/release/ci-conductor/src/quarantine.test.ts b/release/ci-conductor/src/quarantine.test.ts
new file mode 100644
index 00000000000..6c35af5f332
--- /dev/null
+++ b/release/ci-conductor/src/quarantine.test.ts
@@ -0,0 +1,244 @@
+import { describe, expect, it } from "bun:test";
+
+import type { NormalizedTest } from "./contract.ts";
+import {
+ type FailedTest,
+ type GateResult,
+ type QuarantineEntry,
+ checkQuarantineGate,
+ compareFailedToQuarantine,
+ junitFailuresToFailedTests,
+ matchKey,
+} from "./quarantine.ts";
+
+const failed = (
+ test_name: string,
+ test_path: string | null = "Suite",
+ file_path: string | null = "e2e/test/foo.cy.spec.ts",
+): FailedTest => ({ test_name, test_path, file_path });
+
+const quarantined = (
+ test_name: string,
+ test_path = "Suite",
+ file_path = "e2e/test/foo.cy.spec.ts",
+): QuarantineEntry => ({ test_name, test_path, file_path, test_suite: "e2e" });
+
+describe("compareFailedToQuarantine", () => {
+ it("puts every failure in `unquarantined` when nothing is quarantined", () => {
+ const failures = [failed("a"), failed("b")];
+
+ const { quarantined: q, unquarantined } = compareFailedToQuarantine(
+ failures,
+ [],
+ );
+
+ expect(q).toEqual([]);
+ expect(unquarantined).toEqual(failures);
+ });
+
+ it("puts every failure in `quarantined` when all are listed", () => {
+ const failures = [failed("a"), failed("b")];
+
+ const { quarantined: q, unquarantined } = compareFailedToQuarantine(
+ failures,
+ [quarantined("a"), quarantined("b")],
+ );
+
+ expect(q).toEqual(failures);
+ expect(unquarantined).toEqual([]);
+ });
+
+ it("partitions a mixed set", () => {
+ const a = failed("a");
+ const b = failed("b");
+ const c = failed("c");
+
+ const { quarantined: q, unquarantined } = compareFailedToQuarantine(
+ [a, b, c],
+ [quarantined("a"), quarantined("c")],
+ );
+
+ expect(q).toEqual([a, c]);
+ expect(unquarantined).toEqual([b]);
+ });
+
+ it("returns two empty buckets for no failures", () => {
+ expect(compareFailedToQuarantine([], [quarantined("a")])).toEqual({
+ quarantined: [],
+ unquarantined: [],
+ });
+ });
+
+ it("matches on all three fields — same name, different file is not a match", () => {
+ const failure = failed("a", "Suite", "e2e/test/foo.cy.spec.ts");
+
+ const { unquarantined } = compareFailedToQuarantine(
+ [failure],
+ [quarantined("a", "Suite", "e2e/test/bar.cy.spec.ts")],
+ );
+
+ expect(unquarantined).toEqual([failure]);
+ });
+
+ it("matches on all three fields — same name/file, different describe path is not a match", () => {
+ const failure = failed("a", "Suite > inner");
+
+ const { unquarantined } = compareFailedToQuarantine(
+ [failure],
+ [quarantined("a", "Suite > other")],
+ );
+
+ expect(unquarantined).toEqual([failure]);
+ });
+
+ it("treats a null path on the failure as an empty string for matching", () => {
+ const failure = failed("a", null, null);
+
+ const { quarantined: q } = compareFailedToQuarantine(
+ [failure],
+ [quarantined("a", "", "")],
+ );
+
+ expect(q).toEqual([failure]);
+ });
+});
+
+describe("matchKey", () => {
+ it("normalizes null/undefined paths to empty strings", () => {
+ expect(
+ matchKey({ filePath: null, testPath: undefined, testName: "a" }),
+ ).toBe(matchKey({ filePath: "", testPath: "", testName: "a" }));
+ });
+
+ it("does not collide when a boundary between fields shifts", () => {
+ // ["a", "b", "c"] vs ["ab", "", "c"] must stay distinct.
+ expect(
+ matchKey({ filePath: "a", testPath: "b", testName: "c" }),
+ ).not.toBe(matchKey({ filePath: "ab", testPath: "", testName: "c" }));
+ });
+});
+
+describe("junitFailuresToFailedTests", () => {
+ it("renames name/path/file to the gate's identity fields", () => {
+ const tests: NormalizedTest[] = [
+ {
+ name: "renders the table",
+ path: "metabase.viz-test",
+ file: "viz_test.clj",
+ status: "failure",
+ },
+ ];
+
+ expect(junitFailuresToFailedTests(tests)).toEqual([
+ {
+ test_name: "renders the table",
+ test_path: "metabase.viz-test",
+ file_path: "viz_test.clj",
+ },
+ ]);
+ });
+
+ it("defaults a missing path/file to null", () => {
+ const tests: NormalizedTest[] = [{ name: "a", status: "failure" }];
+
+ expect(junitFailuresToFailedTests(tests)).toEqual([
+ { test_name: "a", test_path: null, file_path: null },
+ ]);
+ });
+
+ it("treats an absent status as a failure (JUnit only emits failures)", () => {
+ const tests: NormalizedTest[] = [{ name: "a" }];
+
+ expect(junitFailuresToFailedTests(tests)).toHaveLength(1);
+ });
+
+ it("drops a non-failure row defensively", () => {
+ const tests: NormalizedTest[] = [
+ { name: "broke", status: "failure" },
+ { name: "recovered", status: "passed" },
+ ];
+
+ expect(junitFailuresToFailedTests(tests)).toEqual([
+ { test_name: "broke", test_path: null, file_path: null },
+ ]);
+ });
+});
+
+describe("checkQuarantineGate", () => {
+ const failure: FailedTest = {
+ test_name: "renders",
+ test_path: "Suite",
+ file_path: "foo.spec.ts",
+ };
+
+ // Both branches under test return before any fetch, so no network is touched.
+ // Capture the gate's log lines so we can assert on the printed verdict.
+ async function runCapturingLogs(
+ opts: Parameters<typeof checkQuarantineGate>[0],
+ ): Promise<{ result: GateResult; verdict: string | undefined }> {
+ const lines: string[] = [];
+ const original = console.log;
+ console.log = (...args: unknown[]) => {
+ lines.push(args.map(String).join(" "));
+ };
+ try {
+ const result = await checkQuarantineGate(opts);
+ return { result, verdict: lines.find((l) => l.includes("VERDICT:")) };
+ } finally {
+ console.log = original;
+ }
+ }
+
+ it("passes without a verdict line when there are no failures to gate", async () => {
+ const { result, verdict } = await runCapturingLogs({
+ suite: "e2e",
+ failures: [],
+ baseUrl: undefined,
+ secret: undefined,
+ dryRun: true,
+ });
+ expect(result).toEqual({
+ shouldFail: false,
+ enforced: false,
+ reason: "no failures to gate",
+ });
+ expect(verdict).toBeUndefined();
+ });
+
+ it("reports COULD NOT CHECK — not FAIL — when the base URL is unset", async () => {
+ const { result, verdict } = await runCapturingLogs({
+ suite: "e2e",
+ failures: [failure],
+ baseUrl: undefined,
+ secret: undefined,
+ dryRun: true,
+ });
+ expect(result.reason).toBe("could not fetch the quarantine list");
+ expect(verdict).toContain("COULD NOT CHECK");
+ expect(verdict).not.toContain("FAIL");
+ });
+
+ it("fails closed but stays unenforced under dry run when it can't check", async () => {
+ const { result } = await runCapturingLogs({
+ suite: "e2e",
+ failures: [failure],
+ baseUrl: undefined,
+ secret: undefined,
+ dryRun: true,
+ });
+ expect(result.shouldFail).toBe(true);
+ expect(result.enforced).toBe(false);
+ });
+
+ it("fails closed AND enforces when it can't check outside dry run", async () => {
+ const { result } = await runCapturingLogs({
+ suite: "e2e",
+ failures: [failure],
+ baseUrl: undefined,
+ secret: undefined,
+ dryRun: false,
+ });
+ expect(result.shouldFail).toBe(true);
+ expect(result.enforced).toBe(true);
+ });
+});
diff --git a/release/ci-conductor/src/quarantine.ts b/release/ci-conductor/src/quarantine.ts
new file mode 100644
index 00000000000..1d5ce06ee37
--- /dev/null
+++ b/release/ci-conductor/src/quarantine.ts
@@ -0,0 +1,265 @@
+// Shared quarantine gate for all three suites: given a run's failed tests, pass
+// iff every failure is on ci-conductor's quarantine list.
+
+import type { NormalizedTest } from "./contract.ts";
+import { log } from "./util.ts";
+
+// The list endpoint can be slow, but the gate must never hang a CI job.
+const REQUEST_TIMEOUT_MS = 15_000;
+
+/** One quarantined test as served by ci-conductor's `/api/quarantine`. */
+export type QuarantineEntry = {
+ test_name: string;
+ test_suite: string;
+ test_path: string;
+ file_path: string;
+};
+
+/**
+ * A test that ultimately failed in this run. The three identity fields mirror
+ * ci-conductor's wire/storage names (`NormalizedTest`'s `name`/`path`/`file`
+ * map to `test_name`/`test_path`/`file_path`); `junitFailuresToFailedTests`
+ * does that rename for the JUnit suites.
+ */
+export type FailedTest = {
+ test_name: string;
+ test_path: string | null;
+ file_path: string | null;
+};
+
+/** The gate's outcome. `enforced` is the raw verdict gated by dry-run. */
+export type GateResult = {
+ /** True when any failure isn't quarantined (or the list couldn't be read). */
+ shouldFail: boolean;
+ /** `shouldFail` AND not a dry run — i.e. the entrypoint should exit non-zero. */
+ enforced: boolean;
+ reason: string;
+};
+
+/**
+ * Identity key for a test: the spec/source file, the describe/namespace path,
+ * and the leaf test name as a JSON tuple. JSON-encoding keeps the parts
+ * distinct, so tuples that differ only in where a boundary falls (`["a","b"]`
+ * vs `["ab",""]`) can't collide.
+ */
+export function matchKey(opts: {
+ filePath: string | null | undefined;
+ testPath: string | null | undefined;
+ testName: string;
+}): string {
+ const { filePath, testPath, testName } = opts;
+ return JSON.stringify([filePath ?? "", testPath ?? "", testName]);
+}
+
+/**
+ * Partition the run's failed tests into those that are quarantined and those
+ * that are not, matching on exact {file_path, test_path, test_name} identity.
+ * A single pass over the failures.
+ */
+export function compareFailedToQuarantine(
+ failedTests: FailedTest[],
+ quarantineEntries: QuarantineEntry[],
+): { quarantined: FailedTest[]; unquarantined: FailedTest[] } {
+ const quarantinedKeys = new Set(
+ quarantineEntries.map((q) =>
+ matchKey({
+ filePath: q.file_path,
+ testPath: q.test_path,
+ testName: q.test_name,
+ }),
+ ),
+ );
+ const quarantined: FailedTest[] = [];
+ const unquarantined: FailedTest[] = [];
+ for (const test of failedTests) {
+ const isQuarantined = quarantinedKeys.has(
+ matchKey({
+ filePath: test.file_path,
+ testPath: test.test_path,
+ testName: test.test_name,
+ }),
+ );
+ (isQuarantined ? quarantined : unquarantined).push(test);
+ }
+ return { quarantined, unquarantined };
+}
+
+/**
+ * Adapt the JUnit suites' normalized failures to the gate's `FailedTest` shape.
+ * `parseJunit` only emits failing/erroring testcases, but we filter on `status`
+ * defensively so a future "passed" row can't leak into the gate.
+ */
+export function junitFailuresToFailedTests(
+ tests: NormalizedTest[],
+): FailedTest[] {
+ return tests
+ .filter((test) => (test.status ?? "failure") === "failure")
+ .map((test) => ({
+ test_name: test.name,
+ test_path: test.path ?? null,
+ file_path: test.file ?? null,
+ }));
+}
+
+/**
+ * Fetch the quarantine list for `suite` from ci-conductor. The reporter POSTs to
+ * `.../webhooks/failed-tests`; the list lives at `.../api/quarantine` on the same
+ * host. Returns null (not []) when it can't be retrieved, so the caller can tell
+ * "nothing quarantined" from "couldn't check". Never throws. Logs the path and
+ * suite only — never the host (public repo) or the secret.
+ */
+export async function fetchQuarantineList(opts: {
+ baseUrl: string;
+ suite: string;
+ secret?: string;
+}): Promise<QuarantineEntry[] | null> {
+ const base = opts.baseUrl.replace(/\/+$/, "");
+ const url = `${base}/api/quarantine?suite=${encodeURIComponent(opts.suite)}`;
+ try {
+ const headers: Record<string, string> = {};
+ if (opts.secret) {
+ headers["x-internal-secret"] = opts.secret;
+ }
+ const response = await fetch(url, {
+ headers,
+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+ });
+ if (!response.ok) {
+ log(
+ `🛑 GET /api/quarantine?suite=${opts.suite} → ${response.status} ${response.statusText}`,
+ );
+ return null;
+ }
+ const body = (await response.json()) as { tests?: QuarantineEntry[] };
+ return body.tests ?? [];
+ } catch (error) {
+ console.error(
+ "[ci-conductor] failed to fetch the quarantine list",
+ error,
+ );
+ return null;
+ }
+}
+
+const RULE = "─".repeat(60);
+
+/** `test_path › test_name`, or just the name when there's no path. */
+function title(test: FailedTest): string {
+ return test.test_path ? `${test.test_path} › ${test.test_name}` : test.test_name;
+}
+
+/** Print the verdict (+ dry-run footer) and return the gate result. */
+function finish(opts: {
+ shouldFail: boolean;
+ reason: string;
+ dryRun: boolean;
+ // When the gate couldn't reach a real pass/fail decision — e.g. the quarantine
+ // list was unreachable — show a distinct "couldn't check" verdict rather than a
+ // red FAIL. It still fails closed (shouldFail stays true), but it isn't a real
+ // gate failure, so it shouldn't read as one in the logs.
+ inconclusive?: boolean;
+}): GateResult {
+ const { shouldFail, reason, dryRun, inconclusive = false } = opts;
+ if (inconclusive) {
+ log(`⚠️ VERDICT: COULD NOT CHECK — ${reason}.`);
+ } else if (shouldFail) {
+ log(`🔴 VERDICT: FAIL — ${reason}.`);
+ } else {
+ log(`🟢 VERDICT: PASS — ${reason}.`);
+ }
+ if (dryRun && shouldFail) {
+ log("🌥️ dry run: not enforced — exiting 0.");
+ }
+ log(RULE);
+ return { shouldFail, enforced: shouldFail && !dryRun, reason };
+}
+
+/**
+ * Check one suite's failures against the quarantine gate: print a readable,
+ * dry-run-aware report of which failures are quarantined and return the verdict.
+ * Pure w.r.t. process state — it computes and returns a `GateResult` but never
+ * touches the exit code; `applyQuarantineGate` is the adapter that enacts it.
+ * Source-agnostic — each suite's entrypoint resolves its own `failures` and
+ * `suite`. Never throws.
+ */
+export async function checkQuarantineGate(opts: {
+ suite: string;
+ failures: FailedTest[];
+ baseUrl: string | undefined;
+ secret: string | undefined;
+ dryRun: boolean;
+}): Promise<GateResult> {
+ const { suite, failures, baseUrl, secret, dryRun } = opts;
+
+ log(RULE);
+ log(`🛡️ CI Conductor quarantine gate · suite "${suite}"`);
+ log(
+ dryRun
+ ? "🌥️ DRY RUN — observing only; this gate never fails the job."
+ : "⚔️ ENFORCING — an unquarantined failure will fail this job.",
+ );
+ log(RULE);
+
+ if (failures.length === 0) {
+ log("✅ no failures this run — nothing to gate.");
+ log(RULE);
+ return { shouldFail: false, enforced: false, reason: "no failures to gate" };
+ }
+
+ if (!baseUrl) {
+ log("⚠️ CI_CONDUCTOR_BASE_URL not set — can't fetch the quarantine list (local run or missing secret).");
+ return finish({
+ shouldFail: true,
+ reason: "could not fetch the quarantine list",
+ dryRun,
+ inconclusive: true,
+ });
+ }
+
+ const list = await fetchQuarantineList({ baseUrl, suite, secret });
+ if (list === null) {
+ // Couldn't read the list ⇒ can't confirm everything is quarantined.
+ return finish({
+ shouldFail: true,
+ reason: "could not fetch the quarantine list",
+ dryRun,
+ inconclusive: true,
+ });
+ }
+
+ const { quarantined, unquarantined } = compareFailedToQuarantine(failures, list);
+
+ log(`📋 quarantine list: ${list.length} test(s) registered for "${suite}"`);
+ log(`💥 this run: ${failures.length} failure(s) to evaluate`);
+ for (const test of quarantined) {
+ log(` 🔒 quarantined ${title(test)}`);
+ log(` ↳ ${test.file_path ?? "(no file)"}`);
+ }
+ for (const test of unquarantined) {
+ log(` 🚨 NOT quarantined ${title(test)}`);
+ log(` ↳ ${test.file_path ?? "(no file)"}`);
+ }
+
+ return finish({
+ shouldFail: unquarantined.length > 0,
+ reason:
+ unquarantined.length > 0
+ ? `${unquarantined.length} of ${failures.length} failure(s) are NOT quarantined`
+ : `all ${failures.length} failure(s) are quarantined`,
+ dryRun,
+ });
+}
+
+/**
+ * Impure adapter over `checkQuarantineGate`: run the check, then apply the
+ * verdict to the process exit code (non-zero only when `enforced`). This is the
+ * one place the gate touches process state, so the engine stays testable.
+ */
+export async function applyQuarantineGate(
+ opts: Parameters<typeof checkQuarantineGate>[0],
+): Promise<void> {
+ const result = await checkQuarantineGate(opts);
+ if (result.enforced) {
+ process.exitCode = 1;
+ }
+}