The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Report backend & driver test failures to ci-conductor (#76439)
Nemanja Glumacfd95704a9cfc
diff --git a/.github/actions/report-failures-to-ci-conductor/action.yml b/.github/actions/report-failures-to-ci-conductor/action.ymlnew file mode 100644index 00000000000..43d87a7b50a--- /dev/null+++ b/.github/actions/report-failures-to-ci-conductor/action.yml@@ -0,0 +1,40 @@+name: Report failures to CI Conductor+description: >-+ Report this job's test failures to ci-conductor. Best-effort; no-ops unless+ CI_CONDUCTOR_BASE_URL is set. Gate it at the call site with continue-on-error+ and an `if:` on the test step's outcome.++inputs:+ test-suite:+ description: Per-job identity label (CI_CONDUCTOR_TEST_SUITE); unique per job/leg.+ required: true+ entrypoint:+ description: bun reporter to run, e.g. release/ci-conductor/src/report-backend.ts.+ required: true++runs:+ using: composite+ steps:+ - name: Detect bun+ id: bun+ shell: bash+ run: |+ if command -v bun >/dev/null; then+ echo "present=true" >> "$GITHUB_OUTPUT"+ else+ echo "present=false" >> "$GITHUB_OUTPUT"+ fi+ - name: Install bun+ if: steps.bun.outputs.present != 'true'+ uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0+ with:+ bun-version: "1.3.14"+ token: ${{ github.token }}+ - name: Report failed tests to ci-conductor+ shell: bash+ env:+ REPO_ID: ${{ github.event.repository.id }}+ COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}+ TARGET_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}+ CI_CONDUCTOR_TEST_SUITE: ${{ inputs.test-suite }}+ run: bun ${{ inputs.entrypoint }}diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.ymlindex 0e23cc91d63..6e1f843a4e4 100644--- a/.github/actions/test-driver/action.yml+++ b/.github/actions/test-driver/action.yml@@ -27,6 +27,10 @@ runs:steps:- name: Prepare back-end environmentuses: ./.github/actions/prepare-backend+ - name: Resolve current job id+ # Resolve JOB_ID early (before the long test step) so the ci-conductor+ # reporter below can link failures to this exact job. Best-effort.+ uses: ./.github/actions/resolve-job-id- name: Prepare front-end environmentif: ${{ inputs.build-static-viz == 'true' }}uses: ./.github/actions/prepare-frontend@@ -44,6 +48,19 @@ runs:# Even if this step fails we let it pass, and then `trunk-io/analytics-uploader` determines# whether to fail or to pass the whole job, based on the quarantine list.continue-on-error: true+ - name: Report failed tests to ci-conductor+ # Positioned right after the test run (before the report/upload steps) so it+ # reads pristine JUnit. Gated to non-clean runs + continue-on-error so it can+ # never change the job outcome — Trunk Quarantine remains the sole arbiter.+ # junit-name is the per-job identity label (e.g. be-tests-postgres-ee); only+ # workflows that also set CI_CONDUCTOR_BASE_URL actually POST. See DEV-2224 /+ # DEV-2235.+ if: ${{ always() && steps.run-test.outcome != 'success' }}+ continue-on-error: true+ uses: ./.github/actions/report-failures-to-ci-conductor+ with:+ test-suite: ${{ inputs.junit-name }}+ entrypoint: release/ci-conductor/src/report-backend.ts- name: Publish Test Report (JUnit)uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0# As per the documentation: https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#steps-contextdiff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.ymlindex 87c0970b051..4e2fc2626f8 100644--- a/.github/workflows/app-db.yml+++ b/.github/workflows/app-db.yml@@ -11,6 +11,13 @@ concurrency:group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-app-dbcancel-in-progress: true+# Exposed to every job so the test-driver action can report failures to+# ci-conductor (secrets: inherit is already set on the app-db call in+# run-tests.yml). See DEV-2224.+env:+ CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}+jobs:be-tests-mariadb:if: ${{ !inputs.skip }}diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.ymlindex fb7ac84d5eb..a4e0a12daac 100644--- a/.github/workflows/backend.yml+++ b/.github/workflows/backend.yml@@ -11,6 +11,13 @@ concurrency:group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-backendcancel-in-progress: true+# Exposed to the test jobs so they can report failures to ci-conductor+# (secrets: inherit is already set on the backend call in run-tests.yml).+# See DEV-2224.+env:+ CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}+jobs:be-linter-clj-kondo:if: ${{ !inputs.skip }}@@ -179,6 +186,11 @@ jobs:env:MB_EDITION: ${{ matrix.job.edition }}+ - name: Resolve current job id+ # Resolve JOB_ID before the long test step so the ci-conductor reporter+ # below can link failures to this exact job. Best-effort.+ uses: ./.github/actions/resolve-job-id+- name: "Test Java ${{ matrix.java-version }} ${{ matrix.job.name }}"id: run-java-testsrun: >-@@ -190,6 +202,18 @@ jobs:# whether to fail or to pass the whole job, based on the quarantine list.continue-on-error: true+ - name: Report failed tests to ci-conductor+ # Positioned between the test run and "Upload Test Results" so it reads+ # pristine JUnit (before Trunk quarantine processing). Gated to non-clean+ # runs + continue-on-error so it can never change the job outcome — Trunk+ # Quarantine remains the sole arbiter. See DEV-2224 / DEV-2235.+ if: ${{ always() && steps.run-java-tests.outcome != 'success' }}+ continue-on-error: true+ uses: ./.github/actions/report-failures-to-ci-conductor+ with:+ test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}+ entrypoint: release/ci-conductor/src/report-backend.ts+- name: Upload Test Resultsuses: ./.github/actions/upload-test-resultsif: always() && (steps.run-java-tests.conclusion != 'skipped' || steps.run-java-tests-21.conclusion != 'skipped')diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex c6a82d31057..c668362c292 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -11,6 +11,13 @@ concurrency:group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-driverscancel-in-progress: true+# Exposed to every driver job so the test-driver action can report failures to+# ci-conductor (secrets: inherit is already set on the drivers call in+# run-tests.yml). See DEV-2224.+env:+ CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}+jobs:# This job determines which driver tests should run based on PR context.# All decision logic is consolidated in mage driver-decisions.diff --git a/.github/workflows/release-tests.yml b/.github/workflows/release-tests.ymlindex b368bdbedad..0fcef7f1397 100644--- a/.github/workflows/release-tests.yml+++ b/.github/workflows/release-tests.yml@@ -65,6 +65,27 @@ jobs:- name: Run release unit testsrun: bun run --cwd release test:unit+ ci-conductor-tests:+ needs: files-changed+ if: needs.files-changed.outputs.release_source == 'true'+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}+ timeout-minutes: 10+ steps:+ - uses: actions/checkout@v6+ with:+ sparse-checkout: "release/ci-conductor"+ - name: Install bun+ uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0+ with:+ bun-version: "1.3.14"+ token: ${{ secrets.GITHUB_TOKEN }}+ - name: Install ci-conductor dependencies+ run: bun install --cwd release/ci-conductor --frozen-lockfile+ - name: Type-check ci-conductor+ run: bun run --cwd release/ci-conductor type-check+ - name: Run ci-conductor unit tests+ run: bun run --cwd release/ci-conductor test+release-test-build:needs: files-changedif: needs.files-changed.outputs.release_source == 'true'diff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.ymlindex d0dbcda1688..2b2badf0d73 100644--- a/.github/workflows/semantic-search.yml+++ b/.github/workflows/semantic-search.yml@@ -11,6 +11,13 @@ concurrency:group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-semantic-searchcancel-in-progress: true+# Exposed to every job so the test-driver action can report failures to+# ci-conductor (secrets: inherit is already set on the semantic-search call in+# run-tests.yml). See DEV-2224.+env:+ CI_CONDUCTOR_BASE_URL: ${{ secrets.CI_CONDUCTOR_BASE_URL }}+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}+jobs:semantic-search-tests-h2:if: ${{ !inputs.skip }}diff --git a/release/ci-conductor/bun.lock b/release/ci-conductor/bun.locknew file mode 100644index 00000000000..fb39148dcf9--- /dev/null+++ b/release/ci-conductor/bun.lock@@ -0,0 +1,25 @@+{+ "lockfileVersion": 1,+ "configVersion": 1,+ "workspaces": {+ "": {+ "name": "ci-conductor",+ "devDependencies": {+ "@types/bun": "^1.3.14",+ "@types/node": "^22.10.0",+ "typescript": "^5.9.3",+ },+ },+ },+ "packages": {+ "@types/bun": ["@types/[email protected]", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="],++ "@types/node": ["@types/[email protected]", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-QWlFW2wf3nTjC13/DqRnBpR4ZO36VJH/JVBkA/vcnmbTBNQIlnObqyqZE1tUR7+Ni23Lda8R1BxMfbXRpCUx5g=="],++ "bun-types": ["[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="],++ "typescript": ["[email protected]", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],++ "undici-types": ["[email protected]", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],+ }+}diff --git a/release/ci-conductor/package.json b/release/ci-conductor/package.jsonnew file mode 100644index 00000000000..0f8d7589529--- /dev/null+++ b/release/ci-conductor/package.json@@ -0,0 +1,18 @@+{+ "name": "ci-conductor",+ "version": "1.0.0",+ "description": "Shared CI Conductor test-failure reporting. Adapter pattern: each test suite (be/fe/e2e) transforms its source data into one canonical shape, and the shared core posts it to ci-conductor. This round ships the backend (JUnit) adapter + the /webhooks/failed-tests transport.",+ "type": "module",+ "private": true,+ "scripts": {+ "test": "bun test",+ "type-check": "tsc --noEmit"+ },+ "author": "Metabase",+ "license": "AGPL-3.0-or-later",+ "devDependencies": {+ "@types/bun": "^1.3.14",+ "@types/node": "^22.10.0",+ "typescript": "^5.9.3"+ }+}diff --git a/release/ci-conductor/src/adapters/backend.ts b/release/ci-conductor/src/adapters/backend.tsnew file mode 100644index 00000000000..6efef3ee2b9--- /dev/null+++ b/release/ci-conductor/src/adapters/backend.ts@@ -0,0 +1,54 @@+// Backend (Clojure/hawk) adapter: hawk's JUnit XML → NormalizedTest[].+//+// hawk always writes JUnit XML to `target/junit/`, so the backend "collector"+// is a post-run scan of that artifact (one runner, one artifact, covering both+// the backend and driver CI paths). The XML parsing itself is shared+// (`../junit.ts`); this owns the backend-specific parts — hawk's `*_test.xml`+// file layout — while the suite label is supplied by the entrypoint.++import { readFileSync, readdirSync } from "node:fs";+import { join } from "node:path";++import type { NormalizedTest } from "../contract.ts";+import { parseJunit } from "../junit.ts";+import { log } from "../util.ts";++const JUNIT_DIR = process.env.JUNIT_DIR || "target/junit";++/** Recursively list `*_test.xml` files under `dir`. Returns [] on any error. */+function findJunitFiles(dir: string): string[] {+ try {+ return readdirSync(dir, { recursive: true })+ .map((entry) => String(entry))+ .filter((entry) => entry.endsWith("_test.xml"))+ .map((entry) => join(dir, entry));+ } catch {+ return [];+ }+}++/**+ * Normalize every JUnit file hawk wrote under `dir` into `NormalizedTest[]` —+ * the source-specific half of the adapter pattern (hawk XML → the shared+ * normalized shape).+ */+export function normalizeBackendJunit(+ dir: string = JUNIT_DIR,+): NormalizedTest[] {+ const files = findJunitFiles(dir);+ const failures = files.flatMap((file) => {+ try {+ return parseJunit(readFileSync(file, "utf8"));+ } catch (error) {+ console.error(`[ci-conductor] failed to read ${file}`, error);+ return [];+ }+ });+ log(+ `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} failing test(s)`,+ );+ for (const test of failures) {+ log(` failing: ${test.path || "(no namespace)"} / ${test.name}`);+ }+ return failures;+}diff --git a/release/ci-conductor/src/contract.ts b/release/ci-conductor/src/contract.tsnew file mode 100644index 00000000000..4467e76775a--- /dev/null+++ b/release/ci-conductor/src/contract.ts@@ -0,0 +1,68 @@+// The normalized shape every adapter produces and every consumer reads.+//+// THE AGREEMENT of the adapter pattern: each suite's normalizer+// (`normalizeBackendJunit` / `normalizeFrontendJunit` / `normalizeCypressFailure`)+// turns its source-specific data into `NormalizedTest[]`, and BOTH downstream+// consumers — `reportTestFailures` (the POST) and, later,+// `checkFailuresAgainstQuarantine` (the gate) — read that exact shape. They MUST+// agree on it: if a test's normalized identity differs between the report path+// and the quarantine path, a failure reported under one name won't match its+// quarantine entry under another. Keeping the shape here, in one place, is what+// prevents that drift.+//+// This mirrors ci-conductor's own contract (server repo `origin/dev`:+// `server/src/webhookTypes.ts` `FailedTestsBody` + `lib/failedTests.ts`+// `resolveTestRow`) — a *superset of mostly-optional* fields, resolved+// server-side per-test → body → column default. So each adapter emits only what+// it truthfully observes (the backend has no per-attempt data; it leaves+// `attempts` absent) and the server fills the rest.+//+// NOTE: runtime validation (zod/valibot) is a deliberate, close follow-up. For+// now the shape is enforced by these types only.++/**+ * One reportable test. The backend (JUnit) normalizer only ever emits+ * `name`/`path`/`file`/`message`/`stack`/`status`; the richer fields+ * (`duration`/`attempts`/`failure_screenshot`) exist for the e2e and fe+ * normalizers that land in later rounds. Field names match the wire contract+ * consumed by ci-conductor's `ingestFailedTests` (`path` → test_path, `file` →+ * file_path, `stack` → stack_trace).+ */+export type NormalizedTest = {+ name: string;+ /** Joined suite/`describe` path. Maps to the server's `test_path`. */+ path?: string | null;+ /** Source file. Maps to the server's `file_path`. */+ file?: string | null;+ message?: string | null;+ stack?: string | null;+ /** Defaults to "failure" server-side when absent. */+ status?: "failure" | "flake" | "passed";+ duration?: number;+ /** Raw per-attempt states, e.g. `[{state:"failed"},{state:"passed"}]`. */+ attempts?: { state: string }[];+ /** Base64 PNG data URI; ci-conductor uploads it and stores the URL. */+ failure_screenshot?: string;+};++/**+ * Run-level context shared by every test in a post, resolved once from the CI+ * environment (see `resolveRunContext` in `transport.ts`). `test_suite` is the+ * per-job identity discriminator. `repo_id`/`run_id`/`job_id` are GitHub+ * *numeric* IDs (FKs in ci-conductor), null when unresolved (all nullable+ * server-side).+ */+export type RunContext = {+ repo_id: number | null;+ run_id: number | null;+ job_id: number | null;+ attempt: number | null;+ test_suite: string;+ sha: string | null;+ target_branch: string | null;+};++/** The full request body: run-level context plus the batch of tests. */+export type FailedTestsBody = RunContext & {+ tests: NormalizedTest[];+};diff --git a/release/ci-conductor/src/junit.test.ts b/release/ci-conductor/src/junit.test.tsnew file mode 100644index 00000000000..dc03ce1f3e8--- /dev/null+++ b/release/ci-conductor/src/junit.test.ts@@ -0,0 +1,191 @@+import { describe, expect, it } from "bun:test";++import { parseJunit } from "./junit.ts";++// A real hawk failure body: first line is `file.clj:line`, then context, then+// the expected/actual assertion. Two <failure>s under one <testcase>.+const TWO_FAILURES = `<?xml version='1.0' encoding='UTF-8'?>+<testsuite name="metabase.agent-lib.representations.repair-test" time="0.039" tests="2" errors="0" failures="2">+<testcase classname="metabase.agent-lib.representations.repair-test" name="idempotency-unwrapped-boolean-wrapper-test" time="0.033" assertions="2">+<failure>+<![CDATA[+repair_test.clj:1184+unwrapping a boolean wrapper stays idempotent+expected: (= [["x" {}]] once)+ actual: (not (= [["x" {}]] ["x"]))+]]>+</failure>+<failure>+<![CDATA[+repair_test.clj:1186+and the result is a fixed point+expected: (= once (repair/repair trivial-mp once))+ actual: (not (= ["x"] ["x" {}]))+]]>+</failure>+</testcase>+</testsuite>`;++// Test names routinely contain unescaped `>`/`<` (Clojure thread arrows `->`,+// comparison preds `>=`; a JS title like "renders > when open"). A JUnit+// producer emits them verbatim in the `name` attribute, and XML allows an+// unescaped `>` inside an attribute value. The parser must not let that `>`+// terminate the tag. Regression for DEV-2224 (these were silently dropped: the+// testcase parsed with an empty name and got skipped).+const ANGLE_BRACKET_NAMES = `<?xml version='1.0' encoding='UTF-8'?>+<testsuite name="metabase.util.cron-test" time="0.063" tests="2" errors="0" failures="2">+<testcase classname="metabase.util.cron-test" name="cron-string->schedule-map-test" time="0.03" assertions="1">+<failure>+<![CDATA[+cron_test.clj:65+nope+]]>+</failure>+</testcase>+<testcase classname="metabase.util.cron-test" name="schedule-map->cron-string-test" time="0.03" assertions="1">+<failure>+<![CDATA[+cron_test.clj:12+also nope+]]>+</failure>+</testcase>+</testsuite>`;++// A passing testcase (self-closing) mixed with an errored one in the same suite.+const PASS_AND_ERROR = `<?xml version='1.0' encoding='UTF-8'?>+<testsuite name="metabase.foo-test" tests="2" errors="1" failures="0">+<testcase classname="metabase.foo-test" name="passing-test" time="0.01" assertions="1"/>+<testcase classname="metabase.foo-test" name="erroring-test" time="0.02">+<error>+<![CDATA[+foo_test.clj:42+boom+java.lang.RuntimeException: boom+]]>+</error>+</testcase>+</testsuite>`;++// Two suites in one document (defensive: also covers a <testsuites> wrapper,+// since we scan testcases regardless of nesting).+const TWO_SUITES = `<?xml version='1.0' encoding='UTF-8'?>+<testsuites>+<testsuite name="metabase.a-test" tests="1" failures="1">+<testcase classname="metabase.a-test" name="a" time="0.01">+<failure><![CDATA[a_test.clj:1+nope+]]></failure>+</testcase>+</testsuite>+<testsuite name="metabase.b-test" tests="1" failures="1">+<testcase classname="metabase.b-test" name="b" time="0.01">+<failure><![CDATA[b_test.clj:2+also nope+]]></failure>+</testcase>+</testsuite>+</testsuites>`;++// hawk with the `message` attribute (https://github.com/metabase/hawk/pull/49):+// the failure carries a first-class `message`, and the body still holds the+// file:line locator + full assertion. The parser must prefer the attribute.+const FAILURE_WITH_MESSAGE = `<?xml version='1.0' encoding='UTF-8'?>+<testsuite name="metabase.foo-test" tests="1" errors="0" failures="1">+<testcase classname="metabase.foo-test" name="bar-test" time="0.01" assertions="1">+<failure message="expected: (= 1 2)">+<![CDATA[+foo_test.clj:42+expected: (= 1 2)+ actual: (not (= 1 2))+]]>+</failure>+</testcase>+</testsuite>`;++// hawk emits `message` on <error> too, alongside the `type` attribute (the+// :error branch of write-assertion-result!* in hawk PR #49). The parser must+// read the message off an <error> the same as a <failure>, ignoring `type`.+const ERROR_WITH_MESSAGE = `<?xml version='1.0' encoding='UTF-8'?>+<testsuite name="metabase.foo-test" tests="1" errors="1" failures="0">+<testcase classname="metabase.foo-test" name="boom-test" time="0.01">+<error type="clojure.lang.ExceptionInfo" message="boom">+<![CDATA[+foo_test.clj:99+java.lang.RuntimeException: boom+]]>+</error>+</testcase>+</testsuite>`;++describe("parseJunit", () => {+ it("prefers the failure `message` attribute over the body", () => {+ const [test] = parseJunit(FAILURE_WITH_MESSAGE);+ expect(test.message).toBe("expected: (= 1 2)");+ // The full trace, including the locator, stays in the stack.+ expect(test.stack).toContain("foo_test.clj:42");+ });++ it("reads the `message` attribute off an <error>, ignoring `type`", () => {+ const [test] = parseJunit(ERROR_WITH_MESSAGE);+ expect(test.message).toBe("boom");+ expect(test.status).toBe("failure");+ expect(test.stack).toContain("java.lang.RuntimeException: boom");+ });++ it("collapses multiple <failure>s in one testcase into a single entry", () => {+ const tests = parseJunit(TWO_FAILURES);+ expect(tests).toHaveLength(1);+ const [test] = tests;+ expect(test.name).toBe("idempotency-unwrapped-boolean-wrapper-test");+ expect(test.path).toBe("metabase.agent-lib.representations.repair-test");+ // file_path is intentionally null — backend identity is (suite, path, name).+ expect(test.file).toBeNull();+ expect(test.status).toBe("failure");+ // No `message` attribute on the <failure> => null (we don't guess from the+ // body; its first line is the file:line locator, not a message).+ expect(test.message).toBeNull();+ // The stack carries both failures.+ expect(test.stack).toContain("repair_test.clj:1184");+ expect(test.stack).toContain("repair_test.clj:1186");+ });++ it("parses test names containing unescaped angle brackets (Clojure `->`)", () => {+ const tests = parseJunit(ANGLE_BRACKET_NAMES);+ expect(tests.map((t) => t.name).sort()).toEqual([+ "cron-string->schedule-map-test",+ "schedule-map->cron-string-test",+ ]);+ expect(tests.every((t) => t.path === "metabase.util.cron-test")).toBe(true);+ expect(tests.every((t) => t.status === "failure")).toBe(true);+ });++ it("reports errored tests as failures and skips passing tests", () => {+ const tests = parseJunit(PASS_AND_ERROR);+ expect(tests).toHaveLength(1);+ expect(tests[0].name).toBe("erroring-test");+ expect(tests[0].file).toBeNull();+ expect(tests[0].status).toBe("failure");+ expect(tests[0].stack).toContain("java.lang.RuntimeException: boom");+ });++ it("handles multiple suites / a <testsuites> wrapper", () => {+ const tests = parseJunit(TWO_SUITES);+ expect(tests.map((t) => t.name).sort()).toEqual(["a", "b"]);+ expect(tests.map((t) => t.path).sort()).toEqual([+ "metabase.a-test",+ "metabase.b-test",+ ]);+ });++ it("returns [] for malformed XML without throwing", () => {+ expect(parseJunit("<not-valid")).toEqual([]);+ });++ it("returns [] for a suite with only passing tests", () => {+ const xml = `<testsuite name="metabase.ok-test" tests="1" failures="0">+<testcase classname="metabase.ok-test" name="ok" time="0.01"/>+</testsuite>`;+ expect(parseJunit(xml)).toEqual([]);+ });+});diff --git a/release/ci-conductor/src/junit.ts b/release/ci-conductor/src/junit.tsnew file mode 100644index 00000000000..553c5763f63--- /dev/null+++ b/release/ci-conductor/src/junit.ts@@ -0,0 +1,115 @@+// The shared JUnit XML parser: a JUnit `<testcase>`/`<failure>` document →+// NormalizedTest[]. Format-level and producer-agnostic — both the backend+// (hawk) and the frontend (jest-junit) adapters build on it. The per-suite+// parts (where the XML lives, how fields map, the suite label) stay in+// `adapters/`. JUnit XML is simple, machine-generated markup, so we parse it+// ourselves rather than pull in a dependency.++import type { NormalizedTest } from "./contract.ts";++/** Decode the XML entities a JUnit producer can emit in attribute values. */+function decodeEntities(value: string): string {+ return value+ .replace(/</g, "<")+ .replace(/>/g, ">")+ .replace(/"/g, '"')+ .replace(/'/g, "'")+ .replace(/&/g, "&");+}++/** Read an attribute off an XML open-tag's attribute string. */+function attr(attrs: string, name: string): string | undefined {+ const match = attrs.match(new RegExp(`\\b${name}\\s*=\\s*"([^"]*)"`));+ return match ? decodeEntities(match[1]) : undefined;+}++/** Unwrap CDATA / decode plain text from a failure/error body. */+function elementBody(inner: string): string {+ const cdata = [...inner.matchAll(/<!\[CDATA\[([\s\S]*?)\]\]>/g)].map((m) =>+ m[1].trim(),+ );+ const text = cdata.length > 0 ? cdata.join("\n") : decodeEntities(inner);+ return text.trim();+}++/**+ * Parse one JUnit XML document into normalized entries — one per `<testcase>`+ * that carries a `<failure>` or `<error>`. Multiple problems in a single+ * testcase are joined into one `stack`. Passing (self-closing or problem-free)+ * testcases are skipped. Never throws.+ *+ * Pure (string → normalized), so it's the unit-tested core that each suite's+ * adapter wraps with its own file discovery / labeling.+ */+export function parseJunit(xml: string): NormalizedTest[] {+ try {+ const tests: NormalizedTest[] = [];+ // <testcase ...>...</testcase> (failing) or <testcase .../> (passing,+ // skipped). classname carries the namespace.+ //+ // Attribute blobs are matched as a run of quoted strings or non-quote chars+ // (`ATTRS`) rather than `[^>]*`, because test names routinely contain an+ // unescaped `>` (Clojure thread arrows `cron-string->schedule-map-test`,+ // `>=`; a JS title like "renders > when open"), and XML doesn't require+ // escaping `>` inside an attribute value. A naive `[^>]` stops at that `>`+ // and silently drops the testcase (it parses with an empty name).+ const ATTRS = `(?:[^>"']|"[^"]*"|'[^']*')*`;+ const testcaseRe = new RegExp(+ `<testcase\\b(${ATTRS}?)(?:/>|>([\\s\\S]*?)</testcase>)`,+ "g",+ );+ const problemRe = new RegExp(+ `<(failure|error)\\b(${ATTRS})>([\\s\\S]*?)</\\1>`,+ "g",+ );+ for (const match of xml.matchAll(testcaseRe)) {+ const attrs = match[1];+ const inner = match[2];+ if (!inner) {+ continue; // self-closing => passed+ }++ const problems = [...inner.matchAll(problemRe)];+ if (problems.length === 0) {+ continue;+ }++ const name = (attr(attrs, "name") || "").trim();+ if (name === "") {+ continue;+ }+ const namespace = (attr(attrs, "classname") || "").trim();++ const stack = problems+ .map((p) => elementBody(p[3]))+ .filter((body) => body !== "")+ .join("\n\n");+ // The `message` attribute on the failure/error is the first-class source+ // of the human-readable message. When it's absent we send null rather+ // than guessing from the body — the body's first line is the `file:line`+ // locator, not a message. The full trace stays in `stack`.+ const message =+ problems+ .map((p) => attr(p[2], "message"))+ .find((m) => m != null && m !== "") ?? null;++ tests.push({+ name,+ path: namespace || undefined,+ // file_path is unstable for backend tests (the stack locator varies by+ // failure mode), so identity is (test_suite, test_path, test_name) and+ // file is sent as null.+ file: null,+ message,+ stack: stack || undefined,+ // JUnit only tells us a test failed/errored, never that it recovered,+ // so everything is "failure".+ status: "failure",+ });+ }+ return tests;+ } catch (error) {+ console.error("[ci-conductor] failed to parse JUnit XML", error);+ return [];+ }+}diff --git a/release/ci-conductor/src/report-backend.ts b/release/ci-conductor/src/report-backend.tsnew file mode 100644index 00000000000..215c272b4f0--- /dev/null+++ b/release/ci-conductor/src/report-backend.ts@@ -0,0 +1,31 @@+#!/usr/bin/env bun++// Backend test-failure reporter entrypoint. Runs as a post-test CI step on the+// backend and driver paths: normalize the raw JUnit hawk just wrote and report+// the failures to ci-conductor. Best-effort — reporting must never break a test+// run.+//+// Run directly with bun (no build step): bun src/report-backend.ts++import { normalizeBackendJunit } from "./adapters/backend.ts";+import { reportTestFailures } from "./transport.ts";+import { log } from "./util.ts";++async function main(): Promise<void> {+ log("backend test-failure reporter starting");+ const env = process.env;+ // `test_suite` is the per-job identity discriminator: each driver / backend+ // job exports its unique label as `CI_CONDUCTOR_TEST_SUITE` (e.g.+ // `be-tests-java-21-ee`, `driver-postgres-ee`) so the same namespace run on+ // different drivers or legs stays distinct in ci-conductor's identity key.+ // Falls back to the driver keyword, then a generic `backend`.+ const testSuite =+ env.CI_CONDUCTOR_TEST_SUITE ||+ (env.DRIVERS ? `driver-${env.DRIVERS}` : "backend");+ await reportTestFailures(normalizeBackendJunit(), testSuite);+}++main().catch((error) => {+ // Last line of defense: reporting must never fail the job.+ console.error("[ci-conductor] backend reporting failed (ignored)", error);+});diff --git a/release/ci-conductor/src/transport.ts b/release/ci-conductor/src/transport.tsnew file mode 100644index 00000000000..540e2b7178a--- /dev/null+++ b/release/ci-conductor/src/transport.ts@@ -0,0 +1,108 @@+// Shared transport: POST a batch of normalized failures to ci-conductor's+// `/webhooks/failed-tests`. Call-site-agnostic — the backend posts once+// post-run; e2e posts per-spec mid-run (later round). Best-effort and never+// throws: reporting must not break a test run.+//+// The quarantine side of the conversation (fetch the list + check failures+// against it) lands here too in a later round — same host, same transport,+// reading the same `NormalizedTest[]` this reports.++import type { NormalizedTest, RunContext } from "./contract.ts";+import { log, toNumber } from "./util.ts";++// The endpoint can be slow, but the reporter must never hang a CI job.+const REQUEST_TIMEOUT_MS = 15_000;++/**+ * Resolve the run-level half of the body from the CI environment. This is the+ * payload's identity, not just log decoration: `repo_id`/`run_id`/`job_id` are+ * GitHub numeric IDs (foreign keys in ci-conductor), `sha`/`target_branch` tie+ * the failures to a commit/PR, and `test_suite` is the per-job dedup+ * discriminator. On PRs the ambient `GITHUB_SHA` is a synthetic merge commit+ * and `GITHUB_BASE_REF` the target, so the workflow sets `COMMIT_SHA`/+ * `TARGET_BRANCH` to the PR's head sha / base ref; we fall back to the ambient+ * vars (push runs, local). `JOB_ID` is exported by the `resolve-job-id`+ * composite action (null when unresolved — the column is nullable).+ */+function resolveRunContext(testSuite: string): RunContext {+ const env = process.env;+ return {+ repo_id: toNumber(env.REPO_ID ?? env.GITHUB_REPOSITORY_ID),+ run_id: toNumber(env.GITHUB_RUN_ID),+ attempt: toNumber(env.GITHUB_RUN_ATTEMPT),+ job_id: toNumber(env.JOB_ID),+ test_suite: testSuite,+ sha: env.COMMIT_SHA || env.GITHUB_SHA || null,+ target_branch: env.TARGET_BRANCH || env.GITHUB_BASE_REF || null,+ };+}++/**+ * Report the given normalized failures to ci-conductor by POSTing them to the+ * webhook under `testSuite`'s run-level context, no-opping when the webhook URL+ * isn't configured (local runs, PRs without the secret). Logs the resolved+ * identity (no secrets, no host) up front so a missing row in ci-conductor can+ * be correlated with — or ruled out against — this exact post. Never throws.+ */+export async function reportTestFailures(+ tests: NormalizedTest[],+ testSuite: string,+): Promise<void> {+ const baseUrl = process.env.CI_CONDUCTOR_BASE_URL;+ const context = resolveRunContext(testSuite);++ log(+ `run context: test_suite=${context.test_suite} sha=${context.sha} ` ++ `run_id=${context.run_id} attempt=${context.attempt} job_id=${context.job_id} ` ++ `repo_id=${context.repo_id} target_branch=${context.target_branch}`,+ );++ if (tests.length === 0) {+ log("no failing tests to report — skipping POST");+ return;+ }+ if (!baseUrl) {+ log(+ "CI_CONDUCTOR_BASE_URL not set — skipping POST (local run or missing secret)",+ );+ return;+ }++ try {+ const body = { ...context, tests };++ const endpoint = `${baseUrl.replace(/\/+$/, "")}/webhooks/failed-tests`;+ const headers: Record<string, string> = {+ "Content-Type": "application/json",+ };+ if (process.env.CI_CONDUCTOR_WEBHOOK_SECRET) {+ // ci-conductor authenticates this endpoint via the x-internal-secret header.+ headers["x-internal-secret"] = process.env.CI_CONDUCTOR_WEBHOOK_SECRET;+ }++ // Path only — never the host (public repo). The secret is never logged.+ log(+ `POSTing ${tests.length} failure(s) to /webhooks/failed-tests ` ++ `(${headers["x-internal-secret"] ? "authenticated" : "no secret header"})`,+ );++ const response = await fetch(endpoint, {+ method: "POST",+ headers,+ body: JSON.stringify(body),+ signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),+ });++ if (!response.ok) {+ console.error(+ `[ci-conductor] failed-tests POST returned ${response.status} ${response.statusText}`,+ );+ } else {+ log(+ `ci-conductor accepted the report (${response.status} ${response.statusText})`,+ );+ }+ } catch (error) {+ console.error("[ci-conductor] failed to POST failed-tests", error);+ }+}diff --git a/release/ci-conductor/src/util.ts b/release/ci-conductor/src/util.tsnew file mode 100644index 00000000000..47a596aa539--- /dev/null+++ b/release/ci-conductor/src/util.ts@@ -0,0 +1,21 @@+// Small shared utilities, kept out of the domain modules (identity / transport /+// adapters) so they don't reach across each other just to borrow a helper.++/** Parse a numeric env var, treating missing/blank/non-numeric as null. */+export function toNumber(value: string | undefined | null): number | null {+ if (value == null || value === "") {+ return null;+ }+ const parsed = Number(value);+ return Number.isFinite(parsed) ? parsed : null;+}++/**+ * Progress line for the CI step log. The reporting step exists largely for+ * visibility, so it narrates what it's doing — but NEVER the webhook secret or+ * the base URL/host (metabase/metabase is a public repo). Only non-sensitive+ * identity fields and the request path are ever logged.+ */+export function log(message: string): void {+ console.log(`[ci-conductor] ${message}`);+}diff --git a/release/ci-conductor/tsconfig.json b/release/ci-conductor/tsconfig.jsonnew file mode 100644index 00000000000..b11713b26f2--- /dev/null+++ b/release/ci-conductor/tsconfig.json@@ -0,0 +1,17 @@+{+ "compilerOptions": {+ "target": "ES2022",+ "module": "ESNext",+ "moduleResolution": "bundler",+ // The source uses explicit `.ts` import specifiers (run directly with bun,+ // never compiled), so tell tsc to accept them. We type-check only — bun is+ // the runtime, so there's no emit.+ "allowImportingTsExtensions": true,+ "noEmit": true,+ "strict": true,+ "esModuleInterop": true,+ "skipLibCheck": true,+ "types": ["bun", "node"]+ },+ "include": ["src/**/*.ts"]+}diff --git a/release/jest.config.cjs b/release/jest.config.cjsindex 2589cc36b39..2ee21eb7d3b 100644--- a/release/jest.config.cjs+++ b/release/jest.config.cjs@@ -2,4 +2,8 @@ module.exports = {testEnvironment: "node",extensionsToTreatAsEsm: [".ts"],transformIgnorePatterns: [],+ // ci-conductor is a separate bun package with its own `bun test`; keep+ // release's jest out of it (its bun:test files can't load under jest, and+ // junit.test.ts otherwise matches `--testPathPattern unit`).+ testPathIgnorePatterns: ["/node_modules/", "<rootDir>/ci-conductor/"],};