Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Report backend & driver test failures to ci-conductor (#76439)

Nemanja Glumacfd95704a9cfc
diff --git a/.github/actions/report-failures-to-ci-conductor/action.yml b/.github/actions/report-failures-to-ci-conductor/action.yml
new file mode 100644
index 00000000000..43d87a7b50a
--- /dev/null
+++ b/.github/actions/report-failures-to-ci-conductor/action.yml
@@ -0,0 +1,40 @@
+name: Report failures to CI Conductor
+description: >-
+ Report this job's test failures to ci-conductor. Best-effort; no-ops unless
+ CI_CONDUCTOR_BASE_URL is set. Gate it at the call site with continue-on-error
+ and an `if:` on the test step's outcome.
+
+inputs:
+ test-suite:
+ description: Per-job identity label (CI_CONDUCTOR_TEST_SUITE); unique per job/leg.
+ required: true
+ entrypoint:
+ description: bun reporter to run, e.g. release/ci-conductor/src/report-backend.ts.
+ required: true
+
+runs:
+ using: composite
+ steps:
+ - name: Detect bun
+ id: bun
+ shell: bash
+ run: |
+ if command -v bun >/dev/null; then
+ echo "present=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "present=false" >> "$GITHUB_OUTPUT"
+ fi
+ - name: Install bun
+ if: steps.bun.outputs.present != 'true'
+ uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+ with:
+ bun-version: "1.3.14"
+ token: ${{ github.token }}
+ - name: Report failed tests to ci-conductor
+ shell: bash
+ env:
+ REPO_ID: ${{ github.event.repository.id }}
+ COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+ TARGET_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
+ CI_CONDUCTOR_TEST_SUITE: ${{ inputs.test-suite }}
+ run: bun ${{ inputs.entrypoint }}
diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.yml
index 0e23cc91d63..6e1f843a4e4 100644
--- a/.github/actions/test-driver/action.yml
+++ b/.github/actions/test-driver/action.yml
@@ -27,6 +27,10 @@ runs:
steps:
- name: Prepare back-end environment
uses: ./.github/actions/prepare-backend
+ - name: Resolve current job id
+ # Resolve JOB_ID early (before the long test step) so the ci-conductor
+ # reporter below can link failures to this exact job. Best-effort.
+ uses: ./.github/actions/resolve-job-id
- name: Prepare front-end environment
if: ${{ inputs.build-static-viz == 'true' }}
uses: ./.github/actions/prepare-frontend
@@ -44,6 +48,19 @@ runs:
# Even if this step fails we let it pass, and then `trunk-io/analytics-uploader` determines
# whether to fail or to pass the whole job, based on the quarantine list.
continue-on-error: true
+ - name: Report failed tests to ci-conductor
+ # Positioned right after the test run (before the report/upload steps) so it
+ # reads pristine JUnit. Gated to non-clean runs + continue-on-error so it can
+ # never change the job outcome — Trunk Quarantine remains the sole arbiter.
+ # junit-name is the per-job identity label (e.g. be-tests-postgres-ee); only
+ # workflows that also set CI_CONDUCTOR_BASE_URL actually POST. See DEV-2224 /
+ # DEV-2235.
+ if: ${{ always() && steps.run-test.outcome != 'success' }}
+ continue-on-error: true
+ uses: ./.github/actions/report-failures-to-ci-conductor
+ with:
+ test-suite: ${{ inputs.junit-name }}
+ entrypoint: release/ci-conductor/src/report-backend.ts
- name: Publish Test Report (JUnit)
uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
# As per the documentation: https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#steps-context
diff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.yml
index 87c0970b051..4e2fc2626f8 100644
--- a/.github/workflows/app-db.yml
+++ b/.github/workflows/app-db.yml
@@ -11,6 +11,13 @@ concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-app-db
cancel-in-progress: true
+# Exposed to every job so the test-driver action can report failures to
+# ci-conductor (secrets: inherit is already set on the app-db call in
+# run-tests.yml). See DEV-2224.
+env:
+ CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}
+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+
jobs:
be-tests-mariadb:
if: ${{ !inputs.skip }}
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index fb7ac84d5eb..a4e0a12daac 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -11,6 +11,13 @@ concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-backend
cancel-in-progress: true
+# Exposed to the test jobs so they can report failures to ci-conductor
+# (secrets: inherit is already set on the backend call in run-tests.yml).
+# See DEV-2224.
+env:
+ CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}
+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+
jobs:
be-linter-clj-kondo:
if: ${{ !inputs.skip }}
@@ -179,6 +186,11 @@ jobs:
env:
MB_EDITION: ${{ matrix.job.edition }}
+ - name: Resolve current job id
+ # Resolve JOB_ID before the long test step so the ci-conductor reporter
+ # below can link failures to this exact job. Best-effort.
+ uses: ./.github/actions/resolve-job-id
+
- name: "Test Java ${{ matrix.java-version }} ${{ matrix.job.name }}"
id: run-java-tests
run: >-
@@ -190,6 +202,18 @@ jobs:
# whether to fail or to pass the whole job, based on the quarantine list.
continue-on-error: true
+ - name: Report failed tests to ci-conductor
+ # Positioned between the test run and "Upload Test Results" so it reads
+ # pristine JUnit (before Trunk quarantine processing). Gated to non-clean
+ # runs + continue-on-error so it can never change the job outcome — Trunk
+ # Quarantine remains the sole arbiter. See DEV-2224 / DEV-2235.
+ if: ${{ always() && steps.run-java-tests.outcome != 'success' }}
+ continue-on-error: true
+ uses: ./.github/actions/report-failures-to-ci-conductor
+ with:
+ test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}
+ entrypoint: release/ci-conductor/src/report-backend.ts
+
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always() && (steps.run-java-tests.conclusion != 'skipped' || steps.run-java-tests-21.conclusion != 'skipped')
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index c6a82d31057..c668362c292 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -11,6 +11,13 @@ concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-drivers
cancel-in-progress: true
+# Exposed to every driver job so the test-driver action can report failures to
+# ci-conductor (secrets: inherit is already set on the drivers call in
+# run-tests.yml). See DEV-2224.
+env:
+ CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}
+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+
jobs:
# This job determines which driver tests should run based on PR context.
# All decision logic is consolidated in mage driver-decisions.
diff --git a/.github/workflows/release-tests.yml b/.github/workflows/release-tests.yml
index b368bdbedad..0fcef7f1397 100644
--- a/.github/workflows/release-tests.yml
+++ b/.github/workflows/release-tests.yml
@@ -65,6 +65,27 @@ jobs:
- name: Run release unit tests
run: bun run --cwd release test:unit
+ ci-conductor-tests:
+ needs: files-changed
+ if: needs.files-changed.outputs.release_source == 'true'
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ sparse-checkout: "release/ci-conductor"
+ - name: Install bun
+ uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+ with:
+ bun-version: "1.3.14"
+ token: ${{ secrets.GITHUB_TOKEN }}
+ - name: Install ci-conductor dependencies
+ run: bun install --cwd release/ci-conductor --frozen-lockfile
+ - name: Type-check ci-conductor
+ run: bun run --cwd release/ci-conductor type-check
+ - name: Run ci-conductor unit tests
+ run: bun run --cwd release/ci-conductor test
+
release-test-build:
needs: files-changed
if: needs.files-changed.outputs.release_source == 'true'
diff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.yml
index d0dbcda1688..2b2badf0d73 100644
--- a/.github/workflows/semantic-search.yml
+++ b/.github/workflows/semantic-search.yml
@@ -11,6 +11,13 @@ concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-semantic-search
cancel-in-progress: true
+# Exposed to every job so the test-driver action can report failures to
+# ci-conductor (secrets: inherit is already set on the semantic-search call in
+# run-tests.yml). See DEV-2224.
+env:
+ CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}
+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+
jobs:
semantic-search-tests-h2:
if: ${{ !inputs.skip }}
diff --git a/release/ci-conductor/bun.lock b/release/ci-conductor/bun.lock
new file mode 100644
index 00000000000..fb39148dcf9
--- /dev/null
+++ b/release/ci-conductor/bun.lock
@@ -0,0 +1,25 @@
+{
+ "lockfileVersion": 1,
+ "configVersion": 1,
+ "workspaces": {
+ "": {
+ "name": "ci-conductor",
+ "devDependencies": {
+ "@types/bun": "^1.3.14",
+ "@types/node": "^22.10.0",
+ "typescript": "^5.9.3",
+ },
+ },
+ },
+ "packages": {
+ "@types/bun": ["@types/[email protected]", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="],
+
+ "@types/node": ["@types/[email protected]", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g=="],
+
+ "bun-types": ["[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="],
+
+ "typescript": ["[email protected]", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
+
+ "undici-types": ["[email protected]", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
+ }
+}
diff --git a/release/ci-conductor/package.json b/release/ci-conductor/package.json
new file mode 100644
index 00000000000..0f8d7589529
--- /dev/null
+++ b/release/ci-conductor/package.json
@@ -0,0 +1,18 @@
+{
+ "name": "ci-conductor",
+ "version": "1.0.0",
+ "description": "Shared CI Conductor test-failure reporting. Adapter pattern: each test suite (be/fe/e2e) transforms its source data into one canonical shape, and the shared core posts it to ci-conductor. This round ships the backend (JUnit) adapter + the /webhooks/failed-tests transport.",
+ "type": "module",
+ "private": true,
+ "scripts": {
+ "test": "bun test",
+ "type-check": "tsc --noEmit"
+ },
+ "author": "Metabase",
+ "license": "AGPL-3.0-or-later",
+ "devDependencies": {
+ "@types/bun": "^1.3.14",
+ "@types/node": "^22.10.0",
+ "typescript": "^5.9.3"
+ }
+}
diff --git a/release/ci-conductor/src/adapters/backend.ts b/release/ci-conductor/src/adapters/backend.ts
new file mode 100644
index 00000000000..6efef3ee2b9
--- /dev/null
+++ b/release/ci-conductor/src/adapters/backend.ts
@@ -0,0 +1,54 @@
+// Backend (Clojure/hawk) adapter: hawk's JUnit XML → NormalizedTest[].
+//
+// hawk always writes JUnit XML to `target/junit/`, so the backend "collector"
+// is a post-run scan of that artifact (one runner, one artifact, covering both
+// the backend and driver CI paths). The XML parsing itself is shared
+// (`../junit.ts`); this owns the backend-specific parts — hawk's `*_test.xml`
+// file layout — while the suite label is supplied by the entrypoint.
+
+import { readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+
+import type { NormalizedTest } from "../contract.ts";
+import { parseJunit } from "../junit.ts";
+import { log } from "../util.ts";
+
+const JUNIT_DIR = process.env.JUNIT_DIR || "target/junit";
+
+/** Recursively list `*_test.xml` files under `dir`. Returns [] on any error. */
+function findJunitFiles(dir: string): string[] {
+ try {
+ return readdirSync(dir, { recursive: true })
+ .map((entry) => String(entry))
+ .filter((entry) => entry.endsWith("_test.xml"))
+ .map((entry) => join(dir, entry));
+ } catch {
+ return [];
+ }
+}
+
+/**
+ * Normalize every JUnit file hawk wrote under `dir` into `NormalizedTest[]` —
+ * the source-specific half of the adapter pattern (hawk XML → the shared
+ * normalized shape).
+ */
+export function normalizeBackendJunit(
+ dir: string = JUNIT_DIR,
+): NormalizedTest[] {
+ const files = findJunitFiles(dir);
+ const failures = files.flatMap((file) => {
+ try {
+ return parseJunit(readFileSync(file, "utf8"));
+ } catch (error) {
+ console.error(`[ci-conductor] failed to read ${file}`, error);
+ return [];
+ }
+ });
+ log(
+ `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} failing test(s)`,
+ );
+ for (const test of failures) {
+ log(` failing: ${test.path || "(no namespace)"} / ${test.name}`);
+ }
+ return failures;
+}
diff --git a/release/ci-conductor/src/contract.ts b/release/ci-conductor/src/contract.ts
new file mode 100644
index 00000000000..4467e76775a
--- /dev/null
+++ b/release/ci-conductor/src/contract.ts
@@ -0,0 +1,68 @@
+// The normalized shape every adapter produces and every consumer reads.
+//
+// THE AGREEMENT of the adapter pattern: each suite's normalizer
+// (`normalizeBackendJunit` / `normalizeFrontendJunit` / `normalizeCypressFailure`)
+// turns its source-specific data into `NormalizedTest[]`, and BOTH downstream
+// consumers — `reportTestFailures` (the POST) and, later,
+// `checkFailuresAgainstQuarantine` (the gate) — read that exact shape. They MUST
+// agree on it: if a test's normalized identity differs between the report path
+// and the quarantine path, a failure reported under one name won't match its
+// quarantine entry under another. Keeping the shape here, in one place, is what
+// prevents that drift.
+//
+// This mirrors ci-conductor's own contract (server repo `origin/dev`:
+// `server/src/webhookTypes.ts` `FailedTestsBody` + `lib/failedTests.ts`
+// `resolveTestRow`) — a *superset of mostly-optional* fields, resolved
+// server-side per-test → body → column default. So each adapter emits only what
+// it truthfully observes (the backend has no per-attempt data; it leaves
+// `attempts` absent) and the server fills the rest.
+//
+// NOTE: runtime validation (zod/valibot) is a deliberate, close follow-up. For
+// now the shape is enforced by these types only.
+
+/**
+ * One reportable test. The backend (JUnit) normalizer only ever emits
+ * `name`/`path`/`file`/`message`/`stack`/`status`; the richer fields
+ * (`duration`/`attempts`/`failure_screenshot`) exist for the e2e and fe
+ * normalizers that land in later rounds. Field names match the wire contract
+ * consumed by ci-conductor's `ingestFailedTests` (`path` → test_path, `file` →
+ * file_path, `stack` → stack_trace).
+ */
+export type NormalizedTest = {
+ name: string;
+ /** Joined suite/`describe` path. Maps to the server's `test_path`. */
+ path?: string | null;
+ /** Source file. Maps to the server's `file_path`. */
+ file?: string | null;
+ message?: string | null;
+ stack?: string | null;
+ /** Defaults to "failure" server-side when absent. */
+ status?: "failure" | "flake" | "passed";
+ duration?: number;
+ /** Raw per-attempt states, e.g. `[{state:"failed"},{state:"passed"}]`. */
+ attempts?: { state: string }[];
+ /** Base64 PNG data URI; ci-conductor uploads it and stores the URL. */
+ failure_screenshot?: string;
+};
+
+/**
+ * Run-level context shared by every test in a post, resolved once from the CI
+ * environment (see `resolveRunContext` in `transport.ts`). `test_suite` is the
+ * per-job identity discriminator. `repo_id`/`run_id`/`job_id` are GitHub
+ * *numeric* IDs (FKs in ci-conductor), null when unresolved (all nullable
+ * server-side).
+ */
+export type RunContext = {
+ repo_id: number | null;
+ run_id: number | null;
+ job_id: number | null;
+ attempt: number | null;
+ test_suite: string;
+ sha: string | null;
+ target_branch: string | null;
+};
+
+/** The full request body: run-level context plus the batch of tests. */
+export type FailedTestsBody = RunContext & {
+ tests: NormalizedTest[];
+};
diff --git a/release/ci-conductor/src/junit.test.ts b/release/ci-conductor/src/junit.test.ts
new file mode 100644
index 00000000000..dc03ce1f3e8
--- /dev/null
+++ b/release/ci-conductor/src/junit.test.ts
@@ -0,0 +1,191 @@
+import { describe, expect, it } from "bun:test";
+
+import { parseJunit } from "./junit.ts";
+
+// A real hawk failure body: first line is `file.clj:line`, then context, then
+// the expected/actual assertion. Two <failure>s under one <testcase>.
+const TWO_FAILURES = `<?xml version='1.0' encoding='UTF-8'?>
+<testsuite name="metabase.agent-lib.representations.repair-test" time="0.039" tests="2" errors="0" failures="2">
+<testcase classname="metabase.agent-lib.representations.repair-test" name="idempotency-unwrapped-boolean-wrapper-test" time="0.033" assertions="2">
+<failure>
+<![CDATA[
+repair_test.clj:1184
+unwrapping a boolean wrapper stays idempotent
+expected: (= [["x" {}]] once)
+ actual: (not (= [["x" {}]] ["x"]))
+]]>
+</failure>
+<failure>
+<![CDATA[
+repair_test.clj:1186
+and the result is a fixed point
+expected: (= once (repair/repair trivial-mp once))
+ actual: (not (= ["x"] ["x" {}]))
+]]>
+</failure>
+</testcase>
+</testsuite>`;
+
+// Test names routinely contain unescaped `>`/`<` (Clojure thread arrows `->`,
+// comparison preds `>=`; a JS title like "renders > when open"). A JUnit
+// producer emits them verbatim in the `name` attribute, and XML allows an
+// unescaped `>` inside an attribute value. The parser must not let that `>`
+// terminate the tag. Regression for DEV-2224 (these were silently dropped: the
+// testcase parsed with an empty name and got skipped).
+const ANGLE_BRACKET_NAMES = `<?xml version='1.0' encoding='UTF-8'?>
+<testsuite name="metabase.util.cron-test" time="0.063" tests="2" errors="0" failures="2">
+<testcase classname="metabase.util.cron-test" name="cron-string->schedule-map-test" time="0.03" assertions="1">
+<failure>
+<![CDATA[
+cron_test.clj:65
+nope
+]]>
+</failure>
+</testcase>
+<testcase classname="metabase.util.cron-test" name="schedule-map->cron-string-test" time="0.03" assertions="1">
+<failure>
+<![CDATA[
+cron_test.clj:12
+also nope
+]]>
+</failure>
+</testcase>
+</testsuite>`;
+
+// A passing testcase (self-closing) mixed with an errored one in the same suite.
+const PASS_AND_ERROR = `<?xml version='1.0' encoding='UTF-8'?>
+<testsuite name="metabase.foo-test" tests="2" errors="1" failures="0">
+<testcase classname="metabase.foo-test" name="passing-test" time="0.01" assertions="1"/>
+<testcase classname="metabase.foo-test" name="erroring-test" time="0.02">
+<error>
+<![CDATA[
+foo_test.clj:42
+boom
+java.lang.RuntimeException: boom
+]]>
+</error>
+</testcase>
+</testsuite>`;
+
+// Two suites in one document (defensive: also covers a <testsuites> wrapper,
+// since we scan testcases regardless of nesting).
+const TWO_SUITES = `<?xml version='1.0' encoding='UTF-8'?>
+<testsuites>
+<testsuite name="metabase.a-test" tests="1" failures="1">
+<testcase classname="metabase.a-test" name="a" time="0.01">
+<failure><![CDATA[a_test.clj:1
+nope
+]]></failure>
+</testcase>
+</testsuite>
+<testsuite name="metabase.b-test" tests="1" failures="1">
+<testcase classname="metabase.b-test" name="b" time="0.01">
+<failure><![CDATA[b_test.clj:2
+also nope
+]]></failure>
+</testcase>
+</testsuite>
+</testsuites>`;
+
+// hawk with the `message` attribute (https://github.com/metabase/hawk/pull/49):
+// the failure carries a first-class `message`, and the body still holds the
+// file:line locator + full assertion. The parser must prefer the attribute.
+const FAILURE_WITH_MESSAGE = `<?xml version='1.0' encoding='UTF-8'?>
+<testsuite name="metabase.foo-test" tests="1" errors="0" failures="1">
+<testcase classname="metabase.foo-test" name="bar-test" time="0.01" assertions="1">
+<failure message="expected: (= 1 2)">
+<![CDATA[
+foo_test.clj:42
+expected: (= 1 2)
+ actual: (not (= 1 2))
+]]>
+</failure>
+</testcase>
+</testsuite>`;
+
+// hawk emits `message` on <error> too, alongside the `type` attribute (the
+// :error branch of write-assertion-result!* in hawk PR #49). The parser must
+// read the message off an <error> the same as a <failure>, ignoring `type`.
+const ERROR_WITH_MESSAGE = `<?xml version='1.0' encoding='UTF-8'?>
+<testsuite name="metabase.foo-test" tests="1" errors="1" failures="0">
+<testcase classname="metabase.foo-test" name="boom-test" time="0.01">
+<error type="clojure.lang.ExceptionInfo" message="boom">
+<![CDATA[
+foo_test.clj:99
+java.lang.RuntimeException: boom
+]]>
+</error>
+</testcase>
+</testsuite>`;
+
+describe("parseJunit", () => {
+ it("prefers the failure `message` attribute over the body", () => {
+ const [test] = parseJunit(FAILURE_WITH_MESSAGE);
+ expect(test.message).toBe("expected: (= 1 2)");
+ // The full trace, including the locator, stays in the stack.
+ expect(test.stack).toContain("foo_test.clj:42");
+ });
+
+ it("reads the `message` attribute off an <error>, ignoring `type`", () => {
+ const [test] = parseJunit(ERROR_WITH_MESSAGE);
+ expect(test.message).toBe("boom");
+ expect(test.status).toBe("failure");
+ expect(test.stack).toContain("java.lang.RuntimeException: boom");
+ });
+
+ it("collapses multiple <failure>s in one testcase into a single entry", () => {
+ const tests = parseJunit(TWO_FAILURES);
+ expect(tests).toHaveLength(1);
+ const [test] = tests;
+ expect(test.name).toBe("idempotency-unwrapped-boolean-wrapper-test");
+ expect(test.path).toBe("metabase.agent-lib.representations.repair-test");
+ // file_path is intentionally null — backend identity is (suite, path, name).
+ expect(test.file).toBeNull();
+ expect(test.status).toBe("failure");
+ // No `message` attribute on the <failure> => null (we don't guess from the
+ // body; its first line is the file:line locator, not a message).
+ expect(test.message).toBeNull();
+ // The stack carries both failures.
+ expect(test.stack).toContain("repair_test.clj:1184");
+ expect(test.stack).toContain("repair_test.clj:1186");
+ });
+
+ it("parses test names containing unescaped angle brackets (Clojure `->`)", () => {
+ const tests = parseJunit(ANGLE_BRACKET_NAMES);
+ expect(tests.map((t) => t.name).sort()).toEqual([
+ "cron-string->schedule-map-test",
+ "schedule-map->cron-string-test",
+ ]);
+ expect(tests.every((t) => t.path === "metabase.util.cron-test")).toBe(true);
+ expect(tests.every((t) => t.status === "failure")).toBe(true);
+ });
+
+ it("reports errored tests as failures and skips passing tests", () => {
+ const tests = parseJunit(PASS_AND_ERROR);
+ expect(tests).toHaveLength(1);
+ expect(tests[0].name).toBe("erroring-test");
+ expect(tests[0].file).toBeNull();
+ expect(tests[0].status).toBe("failure");
+ expect(tests[0].stack).toContain("java.lang.RuntimeException: boom");
+ });
+
+ it("handles multiple suites / a <testsuites> wrapper", () => {
+ const tests = parseJunit(TWO_SUITES);
+ expect(tests.map((t) => t.name).sort()).toEqual(["a", "b"]);
+ expect(tests.map((t) => t.path).sort()).toEqual([
+ "metabase.a-test",
+ "metabase.b-test",
+ ]);
+ });
+
+ it("returns [] for malformed XML without throwing", () => {
+ expect(parseJunit("<not-valid")).toEqual([]);
+ });
+
+ it("returns [] for a suite with only passing tests", () => {
+ const xml = `<testsuite name="metabase.ok-test" tests="1" failures="0">
+<testcase classname="metabase.ok-test" name="ok" time="0.01"/>
+</testsuite>`;
+ expect(parseJunit(xml)).toEqual([]);
+ });
+});
diff --git a/release/ci-conductor/src/junit.ts b/release/ci-conductor/src/junit.ts
new file mode 100644
index 00000000000..553c5763f63
--- /dev/null
+++ b/release/ci-conductor/src/junit.ts
@@ -0,0 +1,115 @@
+// The shared JUnit XML parser: a JUnit `<testcase>`/`<failure>` document →
+// NormalizedTest[]. Format-level and producer-agnostic — both the backend
+// (hawk) and the frontend (jest-junit) adapters build on it. The per-suite
+// parts (where the XML lives, how fields map, the suite label) stay in
+// `adapters/`. JUnit XML is simple, machine-generated markup, so we parse it
+// ourselves rather than pull in a dependency.
+
+import type { NormalizedTest } from "./contract.ts";
+
+/** Decode the XML entities a JUnit producer can emit in attribute values. */
+function decodeEntities(value: string): string {
+ return value
+ .replace(/&lt;/g, "<")
+ .replace(/&gt;/g, ">")
+ .replace(/&quot;/g, '"')
+ .replace(/&apos;/g, "'")
+ .replace(/&amp;/g, "&");
+}
+
+/** Read an attribute off an XML open-tag's attribute string. */
+function attr(attrs: string, name: string): string | undefined {
+ const match = attrs.match(new RegExp(`\\b${name}\\s*=\\s*"([^"]*)"`));
+ return match ? decodeEntities(match[1]) : undefined;
+}
+
+/** Unwrap CDATA / decode plain text from a failure/error body. */
+function elementBody(inner: string): string {
+ const cdata = [...inner.matchAll(/<!\[CDATA\[([\s\S]*?)\]\]>/g)].map((m) =>
+ m[1].trim(),
+ );
+ const text = cdata.length > 0 ? cdata.join("\n") : decodeEntities(inner);
+ return text.trim();
+}
+
+/**
+ * Parse one JUnit XML document into normalized entries — one per `<testcase>`
+ * that carries a `<failure>` or `<error>`. Multiple problems in a single
+ * testcase are joined into one `stack`. Passing (self-closing or problem-free)
+ * testcases are skipped. Never throws.
+ *
+ * Pure (string → normalized), so it's the unit-tested core that each suite's
+ * adapter wraps with its own file discovery / labeling.
+ */
+export function parseJunit(xml: string): NormalizedTest[] {
+ try {
+ const tests: NormalizedTest[] = [];
+ // <testcase ...>...</testcase> (failing) or <testcase .../> (passing,
+ // skipped). classname carries the namespace.
+ //
+ // Attribute blobs are matched as a run of quoted strings or non-quote chars
+ // (`ATTRS`) rather than `[^>]*`, because test names routinely contain an
+ // unescaped `>` (Clojure thread arrows `cron-string->schedule-map-test`,
+ // `>=`; a JS title like "renders > when open"), and XML doesn't require
+ // escaping `>` inside an attribute value. A naive `[^>]` stops at that `>`
+ // and silently drops the testcase (it parses with an empty name).
+ const ATTRS = `(?:[^>"']|"[^"]*"|'[^']*')*`;
+ const testcaseRe = new RegExp(
+ `<testcase\\b(${ATTRS}?)(?:/>|>([\\s\\S]*?)</testcase>)`,
+ "g",
+ );
+ const problemRe = new RegExp(
+ `<(failure|error)\\b(${ATTRS})>([\\s\\S]*?)</\\1>`,
+ "g",
+ );
+ for (const match of xml.matchAll(testcaseRe)) {
+ const attrs = match[1];
+ const inner = match[2];
+ if (!inner) {
+ continue; // self-closing => passed
+ }
+
+ const problems = [...inner.matchAll(problemRe)];
+ if (problems.length === 0) {
+ continue;
+ }
+
+ const name = (attr(attrs, "name") || "").trim();
+ if (name === "") {
+ continue;
+ }
+ const namespace = (attr(attrs, "classname") || "").trim();
+
+ const stack = problems
+ .map((p) => elementBody(p[3]))
+ .filter((body) => body !== "")
+ .join("\n\n");
+ // The `message` attribute on the failure/error is the first-class source
+ // of the human-readable message. When it's absent we send null rather
+ // than guessing from the body — the body's first line is the `file:line`
+ // locator, not a message. The full trace stays in `stack`.
+ const message =
+ problems
+ .map((p) => attr(p[2], "message"))
+ .find((m) => m != null && m !== "") ?? null;
+
+ tests.push({
+ name,
+ path: namespace || undefined,
+ // file_path is unstable for backend tests (the stack locator varies by
+ // failure mode), so identity is (test_suite, test_path, test_name) and
+ // file is sent as null.
+ file: null,
+ message,
+ stack: stack || undefined,
+ // JUnit only tells us a test failed/errored, never that it recovered,
+ // so everything is "failure".
+ status: "failure",
+ });
+ }
+ return tests;
+ } catch (error) {
+ console.error("[ci-conductor] failed to parse JUnit XML", error);
+ return [];
+ }
+}
diff --git a/release/ci-conductor/src/report-backend.ts b/release/ci-conductor/src/report-backend.ts
new file mode 100644
index 00000000000..215c272b4f0
--- /dev/null
+++ b/release/ci-conductor/src/report-backend.ts
@@ -0,0 +1,31 @@
+#!/usr/bin/env bun
+
+// Backend test-failure reporter entrypoint. Runs as a post-test CI step on the
+// backend and driver paths: normalize the raw JUnit hawk just wrote and report
+// the failures to ci-conductor. Best-effort — reporting must never break a test
+// run.
+//
+// Run directly with bun (no build step): bun src/report-backend.ts
+
+import { normalizeBackendJunit } from "./adapters/backend.ts";
+import { reportTestFailures } from "./transport.ts";
+import { log } from "./util.ts";
+
+async function main(): Promise<void> {
+ log("backend test-failure reporter starting");
+ const env = process.env;
+ // `test_suite` is the per-job identity discriminator: each driver / backend
+ // job exports its unique label as `CI_CONDUCTOR_TEST_SUITE` (e.g.
+ // `be-tests-java-21-ee`, `driver-postgres-ee`) so the same namespace run on
+ // different drivers or legs stays distinct in ci-conductor's identity key.
+ // Falls back to the driver keyword, then a generic `backend`.
+ const testSuite =
+ env.CI_CONDUCTOR_TEST_SUITE ||
+ (env.DRIVERS ? `driver-${env.DRIVERS}` : "backend");
+ await reportTestFailures(normalizeBackendJunit(), testSuite);
+}
+
+main().catch((error) => {
+ // Last line of defense: reporting must never fail the job.
+ console.error("[ci-conductor] backend reporting failed (ignored)", error);
+});
diff --git a/release/ci-conductor/src/transport.ts b/release/ci-conductor/src/transport.ts
new file mode 100644
index 00000000000..540e2b7178a
--- /dev/null
+++ b/release/ci-conductor/src/transport.ts
@@ -0,0 +1,108 @@
+// Shared transport: POST a batch of normalized failures to ci-conductor's
+// `/webhooks/failed-tests`. Call-site-agnostic — the backend posts once
+// post-run; e2e posts per-spec mid-run (later round). Best-effort and never
+// throws: reporting must not break a test run.
+//
+// The quarantine side of the conversation (fetch the list + check failures
+// against it) lands here too in a later round — same host, same transport,
+// reading the same `NormalizedTest[]` this reports.
+
+import type { NormalizedTest, RunContext } from "./contract.ts";
+import { log, toNumber } from "./util.ts";
+
+// The endpoint can be slow, but the reporter must never hang a CI job.
+const REQUEST_TIMEOUT_MS = 15_000;
+
+/**
+ * Resolve the run-level half of the body from the CI environment. This is the
+ * payload's identity, not just log decoration: `repo_id`/`run_id`/`job_id` are
+ * GitHub numeric IDs (foreign keys in ci-conductor), `sha`/`target_branch` tie
+ * the failures to a commit/PR, and `test_suite` is the per-job dedup
+ * discriminator. On PRs the ambient `GITHUB_SHA` is a synthetic merge commit
+ * and `GITHUB_BASE_REF` the target, so the workflow sets `COMMIT_SHA`/
+ * `TARGET_BRANCH` to the PR's head sha / base ref; we fall back to the ambient
+ * vars (push runs, local). `JOB_ID` is exported by the `resolve-job-id`
+ * composite action (null when unresolved — the column is nullable).
+ */
+function resolveRunContext(testSuite: string): RunContext {
+ const env = process.env;
+ return {
+ repo_id: toNumber(env.REPO_ID ?? env.GITHUB_REPOSITORY_ID),
+ run_id: toNumber(env.GITHUB_RUN_ID),
+ attempt: toNumber(env.GITHUB_RUN_ATTEMPT),
+ job_id: toNumber(env.JOB_ID),
+ test_suite: testSuite,
+ sha: env.COMMIT_SHA || env.GITHUB_SHA || null,
+ target_branch: env.TARGET_BRANCH || env.GITHUB_BASE_REF || null,
+ };
+}
+
+/**
+ * Report the given normalized failures to ci-conductor by POSTing them to the
+ * webhook under `testSuite`'s run-level context, no-opping when the webhook URL
+ * isn't configured (local runs, PRs without the secret). Logs the resolved
+ * identity (no secrets, no host) up front so a missing row in ci-conductor can
+ * be correlated with — or ruled out against — this exact post. Never throws.
+ */
+export async function reportTestFailures(
+ tests: NormalizedTest[],
+ testSuite: string,
+): Promise<void> {
+ const baseUrl = process.env.CI_CONDUCTOR_BASE_URL;
+ const context = resolveRunContext(testSuite);
+
+ log(
+ `run context: test_suite=${context.test_suite} sha=${context.sha} ` +
+ `run_id=${context.run_id} attempt=${context.attempt} job_id=${context.job_id} ` +
+ `repo_id=${context.repo_id} target_branch=${context.target_branch}`,
+ );
+
+ if (tests.length === 0) {
+ log("no failing tests to report — skipping POST");
+ return;
+ }
+ if (!baseUrl) {
+ log(
+ "CI_CONDUCTOR_BASE_URL not set — skipping POST (local run or missing secret)",
+ );
+ return;
+ }
+
+ try {
+ const body = { ...context, tests };
+
+ const endpoint = `${baseUrl.replace(/\/+$/, "")}/webhooks/failed-tests`;
+ const headers: Record<string, string> = {
+ "Content-Type": "application/json",
+ };
+ if (process.env.CI_CONDUCTOR_WEBHOOK_SECRET) {
+ // ci-conductor authenticates this endpoint via the x-internal-secret header.
+ headers["x-internal-secret"] = process.env.CI_CONDUCTOR_WEBHOOK_SECRET;
+ }
+
+ // Path only — never the host (public repo). The secret is never logged.
+ log(
+ `POSTing ${tests.length} failure(s) to /webhooks/failed-tests ` +
+ `(${headers["x-internal-secret"] ? "authenticated" : "no secret header"})`,
+ );
+
+ const response = await fetch(endpoint, {
+ method: "POST",
+ headers,
+ body: JSON.stringify(body),
+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+ });
+
+ if (!response.ok) {
+ console.error(
+ `[ci-conductor] failed-tests POST returned ${response.status} ${response.statusText}`,
+ );
+ } else {
+ log(
+ `ci-conductor accepted the report (${response.status} ${response.statusText})`,
+ );
+ }
+ } catch (error) {
+ console.error("[ci-conductor] failed to POST failed-tests", error);
+ }
+}
diff --git a/release/ci-conductor/src/util.ts b/release/ci-conductor/src/util.ts
new file mode 100644
index 00000000000..47a596aa539
--- /dev/null
+++ b/release/ci-conductor/src/util.ts
@@ -0,0 +1,21 @@
+// Small shared utilities, kept out of the domain modules (identity / transport /
+// adapters) so they don't reach across each other just to borrow a helper.
+
+/** Parse a numeric env var, treating missing/blank/non-numeric as null. */
+export function toNumber(value: string | undefined | null): number | null {
+ if (value == null || value === "") {
+ return null;
+ }
+ const parsed = Number(value);
+ return Number.isFinite(parsed) ? parsed : null;
+}
+
+/**
+ * Progress line for the CI step log. The reporting step exists largely for
+ * visibility, so it narrates what it's doing — but NEVER the webhook secret or
+ * the base URL/host (metabase/metabase is a public repo). Only non-sensitive
+ * identity fields and the request path are ever logged.
+ */
+export function log(message: string): void {
+ console.log(`[ci-conductor] ${message}`);
+}
diff --git a/release/ci-conductor/tsconfig.json b/release/ci-conductor/tsconfig.json
new file mode 100644
index 00000000000..b11713b26f2
--- /dev/null
+++ b/release/ci-conductor/tsconfig.json
@@ -0,0 +1,17 @@
+{
+ "compilerOptions": {
+ "target": "ES2022",
+ "module": "ESNext",
+ "moduleResolution": "bundler",
+ // The source uses explicit `.ts` import specifiers (run directly with bun,
+ // never compiled), so tell tsc to accept them. We type-check only — bun is
+ // the runtime, so there's no emit.
+ "allowImportingTsExtensions": true,
+ "noEmit": true,
+ "strict": true,
+ "esModuleInterop": true,
+ "skipLibCheck": true,
+ "types": ["bun", "node"]
+ },
+ "include": ["src/**/*.ts"]
+}
diff --git a/release/jest.config.cjs b/release/jest.config.cjs
index 2589cc36b39..2ee21eb7d3b 100644
--- a/release/jest.config.cjs
+++ b/release/jest.config.cjs
@@ -2,4 +2,8 @@ module.exports = {
testEnvironment: "node",
extensionsToTreatAsEsm: [".ts"],
transformIgnorePatterns: [],
+ // ci-conductor is a separate bun package with its own `bun test`; keep
+ // release's jest out of it (its bun:test files can't load under jest, and
+ // junit.test.ts otherwise matches `--testPathPattern unit`).
+ testPathIgnorePatterns: ["/node_modules/", "<rootDir>/ci-conductor/"],
};