The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add a unified, dry-run ci-conductor quarantine gate (#76752)
Nemanja Glumac7098c5576a56
diff --git a/.github/actions/quarantine-gate/action.yml b/.github/actions/quarantine-gate/action.ymlnew file mode 100644index 00000000000..7abf517a78b--- /dev/null+++ b/.github/actions/quarantine-gate/action.yml@@ -0,0 +1,46 @@+name: CI Conductor quarantine gate (dry run)+description: >-+ Check this job's test failures against ci-conductor's quarantine list and print+ a verdict. DRY RUN — it never fails the job. No-ops unless CI_CONDUCTOR_BASE_URL+ is set.++inputs:+ adapter:+ description: Which gate to run — one of `backend`, `frontend`, `e2e`.+ required: true+ test-suite:+ description: >-+ Suite label to gate (CI_CONDUCTOR_TEST_SUITE). MUST match the label this+ job reports under, or the gate checks the wrong quarantine list.+ required: true++runs:+ using: composite+ steps:+ - name: Detect bun+ id: bun+ shell: bash+ run: |+ if command -v bun >/dev/null; then+ echo "present=true" >> "$GITHUB_OUTPUT"+ else+ echo "present=false" >> "$GITHUB_OUTPUT"+ fi+ - name: Install bun+ if: steps.bun.outputs.present != 'true'+ uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0+ with:+ bun-version: "1.3.14"+ token: ${{ github.token }}+ - name: Check failures against the quarantine list+ shell: bash+ env:+ CI_CONDUCTOR_TEST_SUITE: ${{ inputs.test-suite }}+ ADAPTER: ${{ inputs.adapter }}+ QUARANTINE_DRY_RUN: "true"+ run: |+ case "$ADAPTER" in+ backend|frontend|e2e) ;;+ *) echo "::error::quarantine-gate: unknown adapter '$ADAPTER' (expected backend|frontend|e2e)"; exit 1 ;;+ esac+ bun "release/ci-conductor/src/check-${ADAPTER}-quarantine.ts"diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.ymlindex d36bff76e41..9940e6debab 100644--- a/.github/actions/test-driver/action.yml+++ b/.github/actions/test-driver/action.yml@@ -61,6 +61,13 @@ runs:with:test-suite: ${{ inputs.junit-name }}adapter: backend+ - name: Quarantine gate (dry run)+ if: ${{ always() && steps.run-test.outcome != 'success' }}+ continue-on-error: true+ uses: ./.github/actions/quarantine-gate+ with:+ adapter: backend+ test-suite: ${{ inputs.junit-name }}- name: Publish Test Report (JUnit)uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0# As per the documentation: https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#steps-contextdiff --git a/.github/workflows/backend.yml b/.github/workflows/backend.ymlindex ff156abc451..09a7f2a7314 100644--- a/.github/workflows/backend.yml+++ b/.github/workflows/backend.yml@@ -214,6 +214,15 @@ jobs:test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}adapter: backend+ - name: Quarantine gate (dry run)+ if: ${{ always() && steps.run-java-tests.outcome != 'success' }}+ continue-on-error: true+ timeout-minutes: 3+ uses: ./.github/actions/quarantine-gate+ with:+ adapter: backend+ test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}+- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always() && (steps.run-java-tests.conclusion != 'skipped' || steps.run-java-tests-21.conclusion != 'skipped')diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.ymlindex e741d7d1b1b..7726a91c23d 100644--- a/.github/workflows/e2e-test.yml+++ b/.github/workflows/e2e-test.yml@@ -214,20 +214,16 @@ jobs:trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}previous-step-outcome: ${{ inputs.specs && steps.tagged-e2e.outcome || steps.split-e2e.outcome }}- # Quarantine gate (DEV-2082). Compare this job's ultimate test failures- # (recorded by after:spec into ./target/quarantine-failures.jsonl) against- # ci-conductor's quarantine list and decide whether the job should pass:- # pass if every failure is quarantined, fail if any isn't. This mirrors- # what Trunk's analytics-uploader does for us today. Runs in dry-run mode- # for now (QUARANTINE_DRY_RUN defaults to true) — it only prints the- # verdict and never fails the job. `continue-on-error` is belt-and-braces- # while it's observational; drop both it and the dry run to enforce.- name: Quarantine gate (dry run)+ # always(), unlike backend/frontend: the run step is continue-on-error+ # (Trunk governs its outcome), so its result can't gate on real failures.if: always()continue-on-error: true- timeout-minutes: 2- shell: bash- run: bun e2e/support/check-quarantine.ts+ timeout-minutes: 3+ uses: ./.github/actions/quarantine-gate+ with:+ adapter: e2e+ test-suite: e2e- name: Upload Cypress Artifacts upon failureuses: actions/upload-artifact@v7diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.ymlindex 15bfcbc00b0..23bd979fb4b 100644--- a/.github/workflows/frontend.yml+++ b/.github/workflows/frontend.yml@@ -125,6 +125,14 @@ jobs:with:test-suite: fe-tests-unitadapter: frontend+ - name: Quarantine gate (dry run)+ if: ${{ always() && steps.fe-tests.outcome != 'success' }}+ continue-on-error: true+ timeout-minutes: 3+ uses: ./.github/actions/quarantine-gate+ with:+ adapter: frontend+ test-suite: fe-tests-unit- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always() && github.event_name != 'workflow_dispatch'diff --git a/e2e/support/check-quarantine.ts b/e2e/support/check-quarantine.tsdeleted file mode 100644index 256d482a10e..00000000000--- a/e2e/support/check-quarantine.ts+++ /dev/null@@ -1,167 +0,0 @@-#!/usr/bin/env bun--// Quarantine gate (DEV-2082). After an e2e job runs, compare the tests that-// *ultimately failed* against the quarantine list served by ci-conductor and-// decide whether the job should pass or fail:-//-// - No failures -> pass (nothing to gate)-// - Every failure is quarantined -> pass (exit cleanly)-// - Any failure is NOT quarantined -> fail-//-// This mirrors what Trunk's analytics-uploader currently does for us; the goal-// is to eventually replace that. For now the gate runs in DRY-RUN mode: it-// computes and prints the verdict but always exits 0, so it can be observed in-// CI without affecting job outcomes. Flip QUARANTINE_DRY_RUN=false to enforce.-//-// The failures come from the file written by after:spec-// (`recordFailedTestsForQuarantine` in ci_conductor.ts). Both that file and-// ci-conductor's quarantine list derive {test_name, test_path, file_path} from-// the same Cypress title array, so the comparison here is exact.--import { readFileSync } from "node:fs";--import {- type FailedTest,- type QuarantineEntry,- compareFailedToQuarantine,-} from "./quarantine-compare";--const {- CI_CONDUCTOR_BASE_URL,- CI_CONDUCTOR_WEBHOOK_SECRET,- // Default to a dry run: compute and print the verdict, but never fail the- // job. Set to "false" to actually gate the build on the verdict.- QUARANTINE_DRY_RUN,- QUARANTINE_FAILURES_FILE,-} = process.env;--const isDryRun = QUARANTINE_DRY_RUN !== "false";--// The test suite to gate. ci-conductor keys its quarantine list by suite.-const TEST_SUITE = "e2e";--const failuresFile =- QUARANTINE_FAILURES_FILE ?? "./target/quarantine-failures.jsonl";--/** Read the run's failed tests from the JSONL file. */-function readFailedTests(file: string): FailedTest[] {- let raw: string;- try {- raw = readFileSync(file, "utf8");- } catch {- console.log(`[quarantine] no failures file at ${file}; nothing to gate.`);- return [];- }-- return raw- .split("\n")- .map((line) => line.trim())- .filter((line) => line !== "")- .map((line): FailedTest | null => {- try {- return JSON.parse(line) as FailedTest;- } catch {- console.error(`[quarantine] skipping unparseable line: ${line}`);- return null;- }- })- .filter((test): test is FailedTest => test !== null);-}--/**- * Build the quarantine list URL from the ci-conductor base origin secret. The- * reporter posts to ".../webhooks/failed-tests"; the quarantine list lives at- * ".../api/quarantine" on the same host.- */-function quarantineUrl(): string | null {- if (!CI_CONDUCTOR_BASE_URL) {- return null;- }- const base = CI_CONDUCTOR_BASE_URL.replace(/\/+$/, "");- return `${base}/api/quarantine?suite=${TEST_SUITE}`;-}--/** Fetch the quarantine list, or null if it can't be retrieved. */-async function fetchQuarantine(): Promise<QuarantineEntry[] | null> {- const url = quarantineUrl();- if (!url) {- console.log(- "[quarantine] CI_CONDUCTOR_BASE_URL is unset; cannot fetch the quarantine list.",- );- return null;- }- try {- const headers: Record<string, string> = {};- if (CI_CONDUCTOR_WEBHOOK_SECRET) {- headers["x-internal-secret"] = CI_CONDUCTOR_WEBHOOK_SECRET;- }- const response = await fetch(url, { headers });- if (!response.ok) {- console.error(- `[quarantine] GET ${url} returned ${response.status} ${response.statusText}`,- );- return null;- }- const body = (await response.json()) as { tests?: QuarantineEntry[] };- return body.tests ?? [];- } catch (error) {- console.error(`[quarantine] failed to fetch ${url}`, error);- return null;- }-}--/** Print the verdict and, outside dry run, set the exit code accordingly. */-function finish(shouldFail: boolean, reason: string): void {- const verdict = shouldFail ? "FAIL" : "PASS";- const mode = isDryRun ? " (dry run — not enforced)" : "";- console.log(`[quarantine] verdict: ${verdict}${mode} — ${reason}.`);- if (shouldFail && !isDryRun) {- process.exitCode = 1;- }-}--async function main(): Promise<void> {- const failed = readFailedTests(failuresFile);-- if (failed.length === 0) {- console.log("[quarantine] no failed tests; passing.");- return;- }-- const quarantine = await fetchQuarantine();- if (quarantine === null) {- // We couldn't read the list, so we can't confirm everything is- // quarantined — that's a failing verdict (enforced only outside dry run).- finish(true, "could not fetch the quarantine list");- return;- }-- const { quarantined, unquarantined } = compareFailedToQuarantine(- failed,- quarantine,- );-- console.log(- `[quarantine] ${failed.length} failed test(s); ${quarantine.length} test(s) in the ${TEST_SUITE} quarantine list.`,- );- const describe = (test: FailedTest) =>- `${test.test_name} (${test.file_path ?? "unknown file"})`;- quarantined.forEach((test) =>- console.log(` 🔒 quarantined: ${describe(test)}`),- );- unquarantined.forEach((test) =>- console.log(` 🚨 NOT quarantined: ${describe(test)}`),- );-- finish(- unquarantined.length > 0,- unquarantined.length > 0- ? `${unquarantined.length} failed test(s) are not quarantined`- : "every failed test is quarantined",- );-}--// Only run when invoked directly (`bun check-quarantine.ts`), not on import.-if (import.meta.main) {- await main();-}diff --git a/e2e/support/ci_conductor.ts b/e2e/support/ci_conductor.tsindex 172e315aa64..60c9dff8a31 100644--- a/e2e/support/ci_conductor.ts+++ b/e2e/support/ci_conductor.ts@@ -267,7 +267,8 @@ export function extractFailedTests(* can read the whole job's failures from a single file. Override with* QUARANTINE_FAILURES_FILE. Note these are the SAME fields ci-conductor stores* in its quarantine list (both derived from the same Cypress title array), so- * the gate can compare them exactly. See `check-quarantine.ts`.+ * the gate can compare them exactly. See+ * `release/ci-conductor/src/check-e2e-quarantine.ts`.*/const QUARANTINE_FAILURES_FILE =process.env.QUARANTINE_FAILURES_FILE ?? "./target/quarantine-failures.jsonl";diff --git a/e2e/support/quarantine-compare.ts b/e2e/support/quarantine-compare.tsdeleted file mode 100644index 13e227a10b5..00000000000--- a/e2e/support/quarantine-compare.ts+++ /dev/null@@ -1,59 +0,0 @@-// Pure comparison logic for the quarantine gate (DEV-2082), split out from the-// `check-quarantine.ts` executable so it can be unit tested without the script's-// CLI side effects (`import.meta.main`, env reads, `fetch`).--import _ from "underscore";--/** One quarantined test as served by ci-conductor's `/api/quarantine`. */-export type QuarantineEntry = {- test_name: string;- test_suite: string;- test_path: string;- file_path: string;-};--/** A test that ultimately failed in this run, as recorded by after:spec. */-export type FailedTest = {- test_name: string;- test_path: string | null;- file_path: string | null;-};--/**- * Identity key for a test: the spec file path, describe path, and leaf test- * name as a JSON tuple. ci-conductor's quarantine list and our after:spec- * recorder both derive these three fields from the same Cypress title array- * (file_path = `spec.relative`, test_path = the joined `describe` titles), so- * they match exactly. JSON-encoding keeps the parts distinct, so tuples that- * differ only in where a boundary falls can't collide.- */-export function matchKey(- filePath: string | null | undefined,- testPath: string | null | undefined,- testName: string,-): string {- return JSON.stringify([filePath ?? "", testPath ?? "", testName]);-}--/**- * Partition the run's failed tests into those that are quarantined and those- * that are not, matching on exact {file_path, test_path, test_name} identity.- * A single pass over the failures, so the gate doesn't repeat the (potentially- * large) comparison when it later logs or counts each bucket.- */-export function compareFailedToQuarantine(- failedTests: FailedTest[],- quarantineEntries: QuarantineEntry[],-): { quarantined: FailedTest[]; unquarantined: FailedTest[] } {- const quarantinedKeys = new Set(- quarantineEntries.map((q) =>- matchKey(q.file_path, q.test_path, q.test_name),- ),- );- const [quarantined, unquarantined] = _.partition(failedTests, (test) =>- quarantinedKeys.has(- matchKey(test.file_path, test.test_path, test.test_name),- ),- );- return { quarantined, unquarantined };-}diff --git a/e2e/support/quarantine-compare.unit.spec.ts b/e2e/support/quarantine-compare.unit.spec.tsdeleted file mode 100644index a5ec8406923..00000000000--- a/e2e/support/quarantine-compare.unit.spec.ts+++ /dev/null@@ -1,114 +0,0 @@-import {- type FailedTest,- type QuarantineEntry,- compareFailedToQuarantine,- matchKey,-} from "./quarantine-compare";--const failed = (- test_name: string,- test_path: string | null = "Suite",- file_path: string | null = "e2e/test/foo.cy.spec.ts",-): FailedTest => ({ test_name, test_path, file_path });--const quarantined = (- test_name: string,- test_path = "Suite",- file_path = "e2e/test/foo.cy.spec.ts",-): QuarantineEntry => ({- test_name,- test_path,- file_path,- test_suite: "e2e",-});--describe("compareFailedToQuarantine", () => {- it("puts every failure in `unquarantined` when nothing is quarantined", () => {- const failures = [failed("a"), failed("b")];-- const { quarantined: q, unquarantined } = compareFailedToQuarantine(- failures,- [],- );-- expect(q).toEqual([]);- expect(unquarantined).toEqual(failures);- });-- it("puts every failure in `quarantined` when all are listed", () => {- const failures = [failed("a"), failed("b")];-- const { quarantined: q, unquarantined } = compareFailedToQuarantine(- failures,- [quarantined("a"), quarantined("b")],- );-- expect(q).toEqual(failures);- expect(unquarantined).toEqual([]);- });-- it("partitions a mixed set", () => {- const a = failed("a");- const b = failed("b");- const c = failed("c");-- const { quarantined: q, unquarantined } = compareFailedToQuarantine(- [a, b, c],- [quarantined("a"), quarantined("c")],- );-- expect(q).toEqual([a, c]);- expect(unquarantined).toEqual([b]);- });-- it("returns two empty buckets for no failures", () => {- expect(compareFailedToQuarantine([], [quarantined("a")])).toEqual({- quarantined: [],- unquarantined: [],- });- });-- it("matches on all three fields — same name, different file is not a match", () => {- const failure = failed("a", "Suite", "e2e/test/foo.cy.spec.ts");-- const { unquarantined } = compareFailedToQuarantine(- [failure],- [quarantined("a", "Suite", "e2e/test/bar.cy.spec.ts")],- );-- expect(unquarantined).toEqual([failure]);- });-- it("matches on all three fields — same name/file, different describe path is not a match", () => {- const failure = failed("a", "Suite > inner");-- const { unquarantined } = compareFailedToQuarantine(- [failure],- [quarantined("a", "Suite > other")],- );-- expect(unquarantined).toEqual([failure]);- });-- it("treats a null path on the failure as an empty string for matching", () => {- const failure = failed("a", null, null);-- const { quarantined: q } = compareFailedToQuarantine(- [failure],- [quarantined("a", "", "")],- );-- expect(q).toEqual([failure]);- });-});--describe("matchKey", () => {- it("normalizes null/undefined paths to empty strings", () => {- expect(matchKey(null, undefined, "a")).toBe(matchKey("", "", "a"));- });-- it("does not collide when a boundary between fields shifts", () => {- // ["a", "b", "c"] vs ["ab", "", "c"] must stay distinct.- expect(matchKey("a", "b", "c")).not.toBe(matchKey("ab", "", "c"));- });-});diff --git a/release/ci-conductor/src/check-backend-quarantine.ts b/release/ci-conductor/src/check-backend-quarantine.tsnew file mode 100644index 00000000000..f1828e00867--- /dev/null+++ b/release/ci-conductor/src/check-backend-quarantine.ts@@ -0,0 +1,31 @@+#!/usr/bin/env bun++import { normalizeBackendJunit } from "./adapters/backend.ts";+import {+ applyQuarantineGate,+ junitFailuresToFailedTests,+} from "./quarantine.ts";+import { log } from "./util.ts";++async function main(): Promise<void> {+ log("backend quarantine gate starting");+ const env = process.env;+ const suite =+ env.CI_CONDUCTOR_TEST_SUITE ||+ (env.DRIVERS ? `driver-${env.DRIVERS}` : "backend");+ await applyQuarantineGate({+ suite,+ failures: junitFailuresToFailedTests(normalizeBackendJunit()),+ baseUrl: env.CI_CONDUCTOR_BASE_URL,+ secret: env.CI_CONDUCTOR_WEBHOOK_SECRET,+ dryRun: env.QUARANTINE_DRY_RUN !== "false",+ });+}++main().catch((error) => {+ // Last line of defense: the gate must never throw into the job.+ console.error(+ "[ci-conductor] backend quarantine gate failed (ignored)",+ error,+ );+});diff --git a/release/ci-conductor/src/check-e2e-quarantine.ts b/release/ci-conductor/src/check-e2e-quarantine.tsnew file mode 100644index 00000000000..38634879d89--- /dev/null+++ b/release/ci-conductor/src/check-e2e-quarantine.ts@@ -0,0 +1,66 @@+#!/usr/bin/env bun++import { readFileSync } from "node:fs";++import { type FailedTest, applyQuarantineGate } from "./quarantine.ts";+import { log } from "./util.ts";++const {+ CI_CONDUCTOR_BASE_URL,+ CI_CONDUCTOR_WEBHOOK_SECRET,+ CI_CONDUCTOR_TEST_SUITE,+ // Default to a dry run: compute and print the verdict, but never fail the job.+ QUARANTINE_DRY_RUN,+ QUARANTINE_FAILURES_FILE,+} = process.env;++const isDryRun = QUARANTINE_DRY_RUN !== "false";++// ci-conductor keys its quarantine list by suite; e2e reports under "e2e".+const TEST_SUITE = CI_CONDUCTOR_TEST_SUITE || "e2e";++const failuresFile =+ QUARANTINE_FAILURES_FILE ?? "./target/quarantine-failures.jsonl";++/** Read the run's ultimate failures from the JSONL file after:spec appended. */+function readFailedTests(file: string): FailedTest[] {+ let raw: string;+ try {+ raw = readFileSync(file, "utf8");+ } catch {+ log(`no failures file at ${file}; nothing to gate.`);+ return [];+ }++ return raw+ .split("\n")+ .map((line) => line.trim())+ .filter((line) => line !== "")+ .map((line): FailedTest | null => {+ try {+ return JSON.parse(line) as FailedTest;+ } catch {+ log(`skipping unparseable line: ${line}`);+ return null;+ }+ })+ .filter((test): test is FailedTest => test !== null);+}++async function main(): Promise<void> {+ await applyQuarantineGate({+ suite: TEST_SUITE,+ failures: readFailedTests(failuresFile),+ baseUrl: CI_CONDUCTOR_BASE_URL,+ secret: CI_CONDUCTOR_WEBHOOK_SECRET,+ dryRun: isDryRun,+ });+}++// Only run when invoked directly (`bun src/check-e2e-quarantine.ts`), not on import.+if (import.meta.main) {+ main().catch((error) => {+ // Last line of defense: the gate must never throw into the job.+ console.error("[ci-conductor] e2e quarantine gate failed (ignored)", error);+ });+}diff --git a/release/ci-conductor/src/check-frontend-quarantine.ts b/release/ci-conductor/src/check-frontend-quarantine.tsnew file mode 100644index 00000000000..60fa4dde5c7--- /dev/null+++ b/release/ci-conductor/src/check-frontend-quarantine.ts@@ -0,0 +1,29 @@+#!/usr/bin/env bun++import { normalizeFrontendJunit } from "./adapters/frontend.ts";+import {+ applyQuarantineGate,+ junitFailuresToFailedTests,+} from "./quarantine.ts";+import { log } from "./util.ts";++async function main(): Promise<void> {+ log("frontend quarantine gate starting");+ const env = process.env;+ const suite = env.CI_CONDUCTOR_TEST_SUITE || "frontend";+ await applyQuarantineGate({+ suite,+ failures: junitFailuresToFailedTests(normalizeFrontendJunit()),+ baseUrl: env.CI_CONDUCTOR_BASE_URL,+ secret: env.CI_CONDUCTOR_WEBHOOK_SECRET,+ dryRun: env.QUARANTINE_DRY_RUN !== "false",+ });+}++main().catch((error) => {+ // Last line of defense: the gate must never throw into the job.+ console.error(+ "[ci-conductor] frontend quarantine gate failed (ignored)",+ error,+ );+});diff --git a/release/ci-conductor/src/quarantine.test.ts b/release/ci-conductor/src/quarantine.test.tsnew file mode 100644index 00000000000..6c35af5f332--- /dev/null+++ b/release/ci-conductor/src/quarantine.test.ts@@ -0,0 +1,244 @@+import { describe, expect, it } from "bun:test";++import type { NormalizedTest } from "./contract.ts";+import {+ type FailedTest,+ type GateResult,+ type QuarantineEntry,+ checkQuarantineGate,+ compareFailedToQuarantine,+ junitFailuresToFailedTests,+ matchKey,+} from "./quarantine.ts";++const failed = (+ test_name: string,+ test_path: string | null = "Suite",+ file_path: string | null = "e2e/test/foo.cy.spec.ts",+): FailedTest => ({ test_name, test_path, file_path });++const quarantined = (+ test_name: string,+ test_path = "Suite",+ file_path = "e2e/test/foo.cy.spec.ts",+): QuarantineEntry => ({ test_name, test_path, file_path, test_suite: "e2e" });++describe("compareFailedToQuarantine", () => {+ it("puts every failure in `unquarantined` when nothing is quarantined", () => {+ const failures = [failed("a"), failed("b")];++ const { quarantined: q, unquarantined } = compareFailedToQuarantine(+ failures,+ [],+ );++ expect(q).toEqual([]);+ expect(unquarantined).toEqual(failures);+ });++ it("puts every failure in `quarantined` when all are listed", () => {+ const failures = [failed("a"), failed("b")];++ const { quarantined: q, unquarantined } = compareFailedToQuarantine(+ failures,+ [quarantined("a"), quarantined("b")],+ );++ expect(q).toEqual(failures);+ expect(unquarantined).toEqual([]);+ });++ it("partitions a mixed set", () => {+ const a = failed("a");+ const b = failed("b");+ const c = failed("c");++ const { quarantined: q, unquarantined } = compareFailedToQuarantine(+ [a, b, c],+ [quarantined("a"), quarantined("c")],+ );++ expect(q).toEqual([a, c]);+ expect(unquarantined).toEqual([b]);+ });++ it("returns two empty buckets for no failures", () => {+ expect(compareFailedToQuarantine([], [quarantined("a")])).toEqual({+ quarantined: [],+ unquarantined: [],+ });+ });++ it("matches on all three fields — same name, different file is not a match", () => {+ const failure = failed("a", "Suite", "e2e/test/foo.cy.spec.ts");++ const { unquarantined } = compareFailedToQuarantine(+ [failure],+ [quarantined("a", "Suite", "e2e/test/bar.cy.spec.ts")],+ );++ expect(unquarantined).toEqual([failure]);+ });++ it("matches on all three fields — same name/file, different describe path is not a match", () => {+ const failure = failed("a", "Suite > inner");++ const { unquarantined } = compareFailedToQuarantine(+ [failure],+ [quarantined("a", "Suite > other")],+ );++ expect(unquarantined).toEqual([failure]);+ });++ it("treats a null path on the failure as an empty string for matching", () => {+ const failure = failed("a", null, null);++ const { quarantined: q } = compareFailedToQuarantine(+ [failure],+ [quarantined("a", "", "")],+ );++ expect(q).toEqual([failure]);+ });+});++describe("matchKey", () => {+ it("normalizes null/undefined paths to empty strings", () => {+ expect(+ matchKey({ filePath: null, testPath: undefined, testName: "a" }),+ ).toBe(matchKey({ filePath: "", testPath: "", testName: "a" }));+ });++ it("does not collide when a boundary between fields shifts", () => {+ // ["a", "b", "c"] vs ["ab", "", "c"] must stay distinct.+ expect(+ matchKey({ filePath: "a", testPath: "b", testName: "c" }),+ ).not.toBe(matchKey({ filePath: "ab", testPath: "", testName: "c" }));+ });+});++describe("junitFailuresToFailedTests", () => {+ it("renames name/path/file to the gate's identity fields", () => {+ const tests: NormalizedTest[] = [+ {+ name: "renders the table",+ path: "metabase.viz-test",+ file: "viz_test.clj",+ status: "failure",+ },+ ];++ expect(junitFailuresToFailedTests(tests)).toEqual([+ {+ test_name: "renders the table",+ test_path: "metabase.viz-test",+ file_path: "viz_test.clj",+ },+ ]);+ });++ it("defaults a missing path/file to null", () => {+ const tests: NormalizedTest[] = [{ name: "a", status: "failure" }];++ expect(junitFailuresToFailedTests(tests)).toEqual([+ { test_name: "a", test_path: null, file_path: null },+ ]);+ });++ it("treats an absent status as a failure (JUnit only emits failures)", () => {+ const tests: NormalizedTest[] = [{ name: "a" }];++ expect(junitFailuresToFailedTests(tests)).toHaveLength(1);+ });++ it("drops a non-failure row defensively", () => {+ const tests: NormalizedTest[] = [+ { name: "broke", status: "failure" },+ { name: "recovered", status: "passed" },+ ];++ expect(junitFailuresToFailedTests(tests)).toEqual([+ { test_name: "broke", test_path: null, file_path: null },+ ]);+ });+});++describe("checkQuarantineGate", () => {+ const failure: FailedTest = {+ test_name: "renders",+ test_path: "Suite",+ file_path: "foo.spec.ts",+ };++ // Both branches under test return before any fetch, so no network is touched.+ // Capture the gate's log lines so we can assert on the printed verdict.+ async function runCapturingLogs(+ opts: Parameters<typeof checkQuarantineGate>[0],+ ): Promise<{ result: GateResult; verdict: string | undefined }> {+ const lines: string[] = [];+ const original = console.log;+ console.log = (...args: unknown[]) => {+ lines.push(args.map(String).join(" "));+ };+ try {+ const result = await checkQuarantineGate(opts);+ return { result, verdict: lines.find((l) => l.includes("VERDICT:")) };+ } finally {+ console.log = original;+ }+ }++ it("passes without a verdict line when there are no failures to gate", async () => {+ const { result, verdict } = await runCapturingLogs({+ suite: "e2e",+ failures: [],+ baseUrl: undefined,+ secret: undefined,+ dryRun: true,+ });+ expect(result).toEqual({+ shouldFail: false,+ enforced: false,+ reason: "no failures to gate",+ });+ expect(verdict).toBeUndefined();+ });++ it("reports COULD NOT CHECK — not FAIL — when the base URL is unset", async () => {+ const { result, verdict } = await runCapturingLogs({+ suite: "e2e",+ failures: [failure],+ baseUrl: undefined,+ secret: undefined,+ dryRun: true,+ });+ expect(result.reason).toBe("could not fetch the quarantine list");+ expect(verdict).toContain("COULD NOT CHECK");+ expect(verdict).not.toContain("FAIL");+ });++ it("fails closed but stays unenforced under dry run when it can't check", async () => {+ const { result } = await runCapturingLogs({+ suite: "e2e",+ failures: [failure],+ baseUrl: undefined,+ secret: undefined,+ dryRun: true,+ });+ expect(result.shouldFail).toBe(true);+ expect(result.enforced).toBe(false);+ });++ it("fails closed AND enforces when it can't check outside dry run", async () => {+ const { result } = await runCapturingLogs({+ suite: "e2e",+ failures: [failure],+ baseUrl: undefined,+ secret: undefined,+ dryRun: false,+ });+ expect(result.shouldFail).toBe(true);+ expect(result.enforced).toBe(true);+ });+});diff --git a/release/ci-conductor/src/quarantine.ts b/release/ci-conductor/src/quarantine.tsnew file mode 100644index 00000000000..1d5ce06ee37--- /dev/null+++ b/release/ci-conductor/src/quarantine.ts@@ -0,0 +1,265 @@+// Shared quarantine gate for all three suites: given a run's failed tests, pass+// iff every failure is on ci-conductor's quarantine list.++import type { NormalizedTest } from "./contract.ts";+import { log } from "./util.ts";++// The list endpoint can be slow, but the gate must never hang a CI job.+const REQUEST_TIMEOUT_MS = 15_000;++/** One quarantined test as served by ci-conductor's `/api/quarantine`. */+export type QuarantineEntry = {+ test_name: string;+ test_suite: string;+ test_path: string;+ file_path: string;+};++/**+ * A test that ultimately failed in this run. The three identity fields mirror+ * ci-conductor's wire/storage names (`NormalizedTest`'s `name`/`path`/`file`+ * map to `test_name`/`test_path`/`file_path`); `junitFailuresToFailedTests`+ * does that rename for the JUnit suites.+ */+export type FailedTest = {+ test_name: string;+ test_path: string | null;+ file_path: string | null;+};++/** The gate's outcome. `enforced` is the raw verdict gated by dry-run. */+export type GateResult = {+ /** True when any failure isn't quarantined (or the list couldn't be read). */+ shouldFail: boolean;+ /** `shouldFail` AND not a dry run — i.e. the entrypoint should exit non-zero. */+ enforced: boolean;+ reason: string;+};++/**+ * Identity key for a test: the spec/source file, the describe/namespace path,+ * and the leaf test name as a JSON tuple. JSON-encoding keeps the parts+ * distinct, so tuples that differ only in where a boundary falls (`["a","b"]`+ * vs `["ab",""]`) can't collide.+ */+export function matchKey(opts: {+ filePath: string | null | undefined;+ testPath: string | null | undefined;+ testName: string;+}): string {+ const { filePath, testPath, testName } = opts;+ return JSON.stringify([filePath ?? "", testPath ?? "", testName]);+}++/**+ * Partition the run's failed tests into those that are quarantined and those+ * that are not, matching on exact {file_path, test_path, test_name} identity.+ * A single pass over the failures.+ */+export function compareFailedToQuarantine(+ failedTests: FailedTest[],+ quarantineEntries: QuarantineEntry[],+): { quarantined: FailedTest[]; unquarantined: FailedTest[] } {+ const quarantinedKeys = new Set(+ quarantineEntries.map((q) =>+ matchKey({+ filePath: q.file_path,+ testPath: q.test_path,+ testName: q.test_name,+ }),+ ),+ );+ const quarantined: FailedTest[] = [];+ const unquarantined: FailedTest[] = [];+ for (const test of failedTests) {+ const isQuarantined = quarantinedKeys.has(+ matchKey({+ filePath: test.file_path,+ testPath: test.test_path,+ testName: test.test_name,+ }),+ );+ (isQuarantined ? quarantined : unquarantined).push(test);+ }+ return { quarantined, unquarantined };+}++/**+ * Adapt the JUnit suites' normalized failures to the gate's `FailedTest` shape.+ * `parseJunit` only emits failing/erroring testcases, but we filter on `status`+ * defensively so a future "passed" row can't leak into the gate.+ */+export function junitFailuresToFailedTests(+ tests: NormalizedTest[],+): FailedTest[] {+ return tests+ .filter((test) => (test.status ?? "failure") === "failure")+ .map((test) => ({+ test_name: test.name,+ test_path: test.path ?? null,+ file_path: test.file ?? null,+ }));+}++/**+ * Fetch the quarantine list for `suite` from ci-conductor. The reporter POSTs to+ * `.../webhooks/failed-tests`; the list lives at `.../api/quarantine` on the same+ * host. Returns null (not []) when it can't be retrieved, so the caller can tell+ * "nothing quarantined" from "couldn't check". Never throws. Logs the path and+ * suite only — never the host (public repo) or the secret.+ */+export async function fetchQuarantineList(opts: {+ baseUrl: string;+ suite: string;+ secret?: string;+}): Promise<QuarantineEntry[] | null> {+ const base = opts.baseUrl.replace(/\/+$/, "");+ const url = `${base}/api/quarantine?suite=${encodeURIComponent(opts.suite)}`;+ try {+ const headers: Record<string, string> = {};+ if (opts.secret) {+ headers["x-internal-secret"] = opts.secret;+ }+ const response = await fetch(url, {+ headers,+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ });+ if (!response.ok) {+ log(+ `🛑 GET /api/quarantine?suite=${opts.suite} → ${response.status} ${response.statusText}`,+ );+ return null;+ }+ const body = (await response.json()) as { tests?: QuarantineEntry[] };+ return body.tests ?? [];+ } catch (error) {+ console.error(+ "[ci-conductor] failed to fetch the quarantine list",+ error,+ );+ return null;+ }+}++const RULE = "─".repeat(60);++/** `test_path › test_name`, or just the name when there's no path. */+function title(test: FailedTest): string {+ return test.test_path ? `${test.test_path} › ${test.test_name}` : test.test_name;+}++/** Print the verdict (+ dry-run footer) and return the gate result. */+function finish(opts: {+ shouldFail: boolean;+ reason: string;+ dryRun: boolean;+ // When the gate couldn't reach a real pass/fail decision — e.g. the quarantine+ // list was unreachable — show a distinct "couldn't check" verdict rather than a+ // red FAIL. It still fails closed (shouldFail stays true), but it isn't a real+ // gate failure, so it shouldn't read as one in the logs.+ inconclusive?: boolean;+}): GateResult {+ const { shouldFail, reason, dryRun, inconclusive = false } = opts;+ if (inconclusive) {+ log(`⚠️ VERDICT: COULD NOT CHECK — ${reason}.`);+ } else if (shouldFail) {+ log(`🔴 VERDICT: FAIL — ${reason}.`);+ } else {+ log(`🟢 VERDICT: PASS — ${reason}.`);+ }+ if (dryRun && shouldFail) {+ log("🌥️ dry run: not enforced — exiting 0.");+ }+ log(RULE);+ return { shouldFail, enforced: shouldFail && !dryRun, reason };+}++/**+ * Check one suite's failures against the quarantine gate: print a readable,+ * dry-run-aware report of which failures are quarantined and return the verdict.+ * Pure w.r.t. process state — it computes and returns a `GateResult` but never+ * touches the exit code; `applyQuarantineGate` is the adapter that enacts it.+ * Source-agnostic — each suite's entrypoint resolves its own `failures` and+ * `suite`. Never throws.+ */+export async function checkQuarantineGate(opts: {+ suite: string;+ failures: FailedTest[];+ baseUrl: string | undefined;+ secret: string | undefined;+ dryRun: boolean;+}): Promise<GateResult> {+ const { suite, failures, baseUrl, secret, dryRun } = opts;++ log(RULE);+ log(`🛡️ CI Conductor quarantine gate · suite "${suite}"`);+ log(+ dryRun+ ? "🌥️ DRY RUN — observing only; this gate never fails the job."+ : "⚔️ ENFORCING — an unquarantined failure will fail this job.",+ );+ log(RULE);++ if (failures.length === 0) {+ log("✅ no failures this run — nothing to gate.");+ log(RULE);+ return { shouldFail: false, enforced: false, reason: "no failures to gate" };+ }++ if (!baseUrl) {+ log("⚠️ CI_CONDUCTOR_BASE_URL not set — can't fetch the quarantine list (local run or missing secret).");+ return finish({+ shouldFail: true,+ reason: "could not fetch the quarantine list",+ dryRun,+ inconclusive: true,+ });+ }++ const list = await fetchQuarantineList({ baseUrl, suite, secret });+ if (list === null) {+ // Couldn't read the list ⇒ can't confirm everything is quarantined.+ return finish({+ shouldFail: true,+ reason: "could not fetch the quarantine list",+ dryRun,+ inconclusive: true,+ });+ }++ const { quarantined, unquarantined } = compareFailedToQuarantine(failures, list);++ log(`📋 quarantine list: ${list.length} test(s) registered for "${suite}"`);+ log(`💥 this run: ${failures.length} failure(s) to evaluate`);+ for (const test of quarantined) {+ log(` 🔒 quarantined ${title(test)}`);+ log(` ↳ ${test.file_path ?? "(no file)"}`);+ }+ for (const test of unquarantined) {+ log(` 🚨 NOT quarantined ${title(test)}`);+ log(` ↳ ${test.file_path ?? "(no file)"}`);+ }++ return finish({+ shouldFail: unquarantined.length > 0,+ reason:+ unquarantined.length > 0+ ? `${unquarantined.length} of ${failures.length} failure(s) are NOT quarantined`+ : `all ${failures.length} failure(s) are quarantined`,+ dryRun,+ });+}++/**+ * Impure adapter over `checkQuarantineGate`: run the check, then apply the+ * verdict to the process exit code (non-zero only when `enforced`). This is the+ * one place the gate touches process state, so the engine stays testable.+ */+export async function applyQuarantineGate(+ opts: Parameters<typeof checkQuarantineGate>[0],+): Promise<void> {+ const result = await checkQuarantineGate(opts);+ if (result.enforced) {+ process.exitCode = 1;+ }+}