The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
report all test results on rerun to CI Conductor (#80326)
Chris Vandoodewaarde30fb0cec1e9
diff --git a/.github/actions/report-failures-to-ci-conductor/action.yml b/.github/actions/report-failures-to-ci-conductor/action.ymlindex 6a0c5245899..b61c2d81a41 100644--- a/.github/actions/report-failures-to-ci-conductor/action.yml+++ b/.github/actions/report-failures-to-ci-conductor/action.yml@@ -25,6 +25,7 @@ runs:TARGET_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}CI_CONDUCTOR_TEST_SUITE: ${{ inputs.test-suite }}ADAPTER: ${{ inputs.adapter }}+ IS_RERUN: ${{ github.run_attempt != '1' }}# Keep the path constrained to known entrypoints.run: |case "$ADAPTER" indiff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.ymlindex 341077a739a..a34f1feb08d 100644--- a/.github/actions/test-driver/action.yml+++ b/.github/actions/test-driver/action.yml@@ -114,7 +114,7 @@ runs:continue-on-error: true- name: Report failed tests to ci-conductor- if: ${{ !cancelled() && steps.run-test.outcome == 'failure' }}+ if: ${{ !cancelled() && ( steps.run-test.outcome == 'failure' || github.run_attempt != '1' ) }}# This step should NEVER fail the job!continue-on-error: trueuses: ./.github/actions/report-failures-to-ci-conductordiff --git a/.github/workflows/backend.yml b/.github/workflows/backend.ymlindex 52039a31896..7cfb922c9bf 100644--- a/.github/workflows/backend.yml+++ b/.github/workflows/backend.yml@@ -232,7 +232,7 @@ jobs:continue-on-error: true- name: Report failed tests to ci-conductor- if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}+ if: ${{ !cancelled() && ( steps.run-java-tests.outcome == 'failure' || github.run_attempt != '1' ) }}# This step should NEVER fail the job!continue-on-error: trueuses: ./.github/actions/report-failures-to-ci-conductordiff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.ymlindex 552b43b6c9a..570cf78b8a8 100644--- a/.github/workflows/frontend.yml+++ b/.github/workflows/frontend.yml@@ -116,7 +116,7 @@ jobs:# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.continue-on-error: true- name: Report failed tests to ci-conductor- if: ${{ !cancelled() && steps.fe-tests.outcome == 'failure' }}+ if: ${{ !cancelled() && ( steps.fe-tests.outcome == 'failure' || github.run_attempt != '1' ) }}# This step should NEVER fail the job!continue-on-error: trueuses: ./.github/actions/report-failures-to-ci-conductordiff --git a/release/ci-conductor/src/adapters/backend.ts b/release/ci-conductor/src/adapters/backend.tsindex 284fce0cd6d..12f27c0549c 100644--- a/release/ci-conductor/src/adapters/backend.ts+++ b/release/ci-conductor/src/adapters/backend.ts@@ -33,22 +33,23 @@ const selectHawkJunit = (entries: string[]): string[] =>* normalized shape).*/export function normalizeBackendJunit(+ ignorePassingTests: boolean = true,dir: string = JUNIT_DIR,): NormalizedTest[] {const files = findJunitFiles(dir, selectHawkJunit);const failures = files.flatMap((file) => {try {- return parseJunit(readFileSync(file, "utf8"));+ return parseJunit(readFileSync(file, "utf8"), ignorePassingTests);} catch (error) {console.error(`[ci-conductor] failed to read ${file}`, error);return [];}});log(- `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} failing test(s)`,+ `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} ${ignorePassingTests ? "failing " : ""}test(s)`,);for (const test of failures) {- log(` failing: ${test.path || "(no namespace)"} / ${test.name}`);+ log(` ${test.status ?? 'failure'}: ${test.path || "(no namespace)"} / ${test.name}`);}return failures;}diff --git a/release/ci-conductor/src/adapters/frontend.ts b/release/ci-conductor/src/adapters/frontend.tsindex 4d7b8efcda0..b8a3a96354f 100644--- a/release/ci-conductor/src/adapters/frontend.ts+++ b/release/ci-conductor/src/adapters/frontend.ts@@ -46,23 +46,24 @@ function selectJestJunit(name: string): (entries: string[]) => string[] {* (jest-junit XML → the shared normalized shape).*/export function normalizeFrontendJunit(+ ignorePassingTests: boolean = true,dir: string = JUNIT_DIR,name: string = JUNIT_NAME,): NormalizedTest[] {const files = findJunitFiles(dir, selectJestJunit(name));const failures = files.flatMap((file) => {try {- return parseJunit(readFileSync(file, "utf8"));+ return parseJunit(readFileSync(file, "utf8"), ignorePassingTests);} catch (error) {console.error(`[ci-conductor] failed to read ${file}`, error);return [];}});log(- `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} failing test(s)`,+ `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} ${ignorePassingTests ? "failing " : ""}test(s)`,);for (const test of failures) {- log(` failing: ${test.path || "(no describe path)"} / ${test.name}`);+ log(` ${test.status ?? 'failure'}: ${test.path || "(no describe path)"} / ${test.name}`);}return failures;}diff --git a/release/ci-conductor/src/junit.test.ts b/release/ci-conductor/src/junit.test.tsindex 8c8b0698b8c..b3bb0339fb6 100644--- a/release/ci-conductor/src/junit.test.ts+++ b/release/ci-conductor/src/junit.test.ts@@ -148,6 +148,48 @@ const JEST_WITH_FILE = `<?xml version="1.0" encoding="UTF-8"?></testsuite></testsuites>`;+// A jest-junit document as it looks on a *rerun*: mostly passes (self-closing+// testcases) with one failure. This is the shape `ignorePassingTests: false`+// exists for — ci-conductor needs the passes to tell a flake (failed, then+// passed on rerun) from a genuine failure.+const JEST_MOSTLY_PASSING = `<?xml version="1.0" encoding="UTF-8"?>+<testsuites name="jest tests" tests="3" failures="1">+<testsuite name="Button" failures="1" tests="3">+<testcase classname="Button renders" name="Button renders" time="0.01" file="frontend/src/metabase/components/Button/Button.unit.spec.tsx"/>+<testcase classname="Button is disabled" name="Button is disabled" time="0.02" file="frontend/src/metabase/components/Button/Button.unit.spec.tsx"/>+<testcase classname="Button explodes" name="Button explodes" time="0.03" file="frontend/src/metabase/components/Button/Button.unit.spec.tsx">+<failure message="expect(received).toBe(expected)">Error: expect(received).toBe(expected)</failure>+</testcase>+</testsuite>+</testsuites>`;++// A testcase that has a body but no <failure>/<error> — jest-junit emits this+// shape when a passing test wrote to stdout. It passed, same as a self-closing+// one.+const PASSING_WITH_BODY = `<?xml version="1.0" encoding="UTF-8"?>+<testsuite name="metabase.foo-test" tests="1" failures="0">+<testcase classname="metabase.foo-test" name="chatty-test" time="0.01">+<system-out>some log output</system-out>+</testcase>+</testsuite>`;++// jest marks a skipped test (`it.skip`/`xit`, or a suite it never ran) with a+// <skipped/> child; a producer may also write it with a `message` attribute or a+// body. A real run mixes skips in among the passes and failures.+const SKIPPED = `<?xml version="1.0" encoding="UTF-8"?>+<testsuite name="Button" tests="4" failures="1" skipped="2">+<testcase classname="Button is todo" name="Button is todo" time="0">+<skipped/>+</testcase>+<testcase classname="Button is deferred" name="Button is deferred" time="0">+<skipped message="pending"/>+</testcase>+<testcase classname="Button renders" name="Button renders" time="0.01"/>+<testcase classname="Button explodes" name="Button explodes" time="0.02">+<failure message="boom">Error: boom</failure>+</testcase>+</testsuite>`;+describe("parseJunit", () => {it("prefers the failure `message` attribute over the body", () => {const [test] = parseJunit(FAILURE_WITH_MESSAGE);@@ -242,6 +284,11 @@ describe("parseJunit", () => {expect(parseJunit("<not-valid")).toEqual([]);});+ it("drops skipped tests, keeping only the failure", () => {+ const tests = parseJunit(SKIPPED);+ expect(tests.map((t) => t.name)).toEqual(["Button explodes"]);+ });+it("returns [] for a suite with only passing tests", () => {const xml = `<testsuite name="metabase.ok-test" tests="1" failures="0"><testcase classname="metabase.ok-test" name="ok" time="0.01"/>@@ -249,3 +296,82 @@ describe("parseJunit", () => {expect(parseJunit(xml)).toEqual([]);});});++describe("parseJunit with ignorePassingTests: false", () => {+ it("emits self-closing (passing) testcases with status `passed`", () => {+ const tests = parseJunit(PASS_AND_ERROR, false);+ expect(tests).toHaveLength(2);+ const passed = tests.find((t) => t.name === "passing-test");+ expect(passed).toBeDefined();+ expect(passed!.status).toBe("passed");+ expect(passed!.path).toBe("metabase.foo-test");+ // Nothing went wrong, so there's no message or stack to carry.+ expect(passed!.message).toBeNull();+ expect(passed!.stack).toBeUndefined();+ });++ it("still reports failures as failures alongside the passes", () => {+ const tests = parseJunit(PASS_AND_ERROR, false);+ const errored = tests.find((t) => t.name === "erroring-test");+ expect(errored).toBeDefined();+ expect(errored!.status).toBe("failure");+ expect(errored!.stack).toContain("java.lang.RuntimeException: boom");+ });++ it("reports every test in a mostly-passing jest rerun, keeping `file`", () => {+ const tests = parseJunit(JEST_MOSTLY_PASSING, false);+ expect(+ tests.map((t) => [t.name, t.status]).sort((a, b) => a[0]!.localeCompare(b[0]!)),+ ).toEqual([+ ["Button explodes", "failure"],+ ["Button is disabled", "passed"],+ ["Button renders", "passed"],+ ]);+ expect(+ tests.every(+ (t) =>+ t.file ===+ "frontend/src/metabase/components/Button/Button.unit.spec.tsx",+ ),+ ).toBe(true);+ });++ it("treats a testcase with a body but no <failure>/<error> as passed", () => {+ const tests = parseJunit(PASSING_WITH_BODY, false);+ expect(tests).toHaveLength(1);+ expect(tests[0].status).toBe("passed");+ expect(parseJunit(PASSING_WITH_BODY)).toEqual([]);+ });++ it("reports a suite of only passing tests instead of []", () => {+ const xml = `<testsuite name="metabase.ok-test" tests="1" failures="0">+<testcase classname="metabase.ok-test" name="ok" time="0.01"/>+</testsuite>`;+ expect(parseJunit(xml, false)).toEqual([+ {+ name: "ok",+ path: "metabase.ok-test",+ file: null,+ message: null,+ stack: undefined,+ status: "passed",+ },+ ]);+ });++ it("still returns [] for malformed XML without throwing", () => {+ expect(parseJunit("<not-valid", false)).toEqual([]);+ });++ it("drops skipped tests while keeping the passes and failures around them", () => {+ const tests = parseJunit(SKIPPED, false);+ expect(+ tests+ .map((t) => [t.name, t.status])+ .sort((a, b) => a[0]!.localeCompare(b[0]!)),+ ).toEqual([+ ["Button explodes", "failure"],+ ["Button renders", "passed"],+ ]);+ });+});diff --git a/release/ci-conductor/src/junit.ts b/release/ci-conductor/src/junit.tsindex 3ea578973a5..4e1d699f703 100644--- a/release/ci-conductor/src/junit.ts+++ b/release/ci-conductor/src/junit.ts@@ -56,15 +56,18 @@ function elementBody(inner: string): string {}/**- * Parse one JUnit XML document into normalized entries — one per `<testcase>`- * that carries a `<failure>` or `<error>`. Multiple problems in a single- * testcase are joined into one `stack`. Passing (self-closing or problem-free)- * testcases are skipped. Never throws.+ * Parse one JUnit XML document into normalized entries. Multiple problems in a+ * single `<testcase>` are joined into one `stack`. Never throws.+ *+ * By default only testcases carrying a `<failure>` or `<error>` are emitted.+ * Pass `ignorePassingTests: false` on a rerun to also emit the passes (as+ * `status: "passed"`), which is what lets ci-conductor tell a recovered flake+ * from a still-broken test. Either way, `<skipped/>` testcases are dropped.** Pure (string → normalized), so it's the unit-tested core that each suite's* adapter wraps with its own file discovery / labeling.*/-export function parseJunit(xml: string): NormalizedTest[] {+export function parseJunit(xml: string, ignorePassingTests: boolean = true): NormalizedTest[] {try {const tests: NormalizedTest[] = [];// <testcase ...>...</testcase> (failing) or <testcase .../> (passing,@@ -85,15 +88,21 @@ export function parseJunit(xml: string): NormalizedTest[] {`<(failure|error)\\b(${ATTRS})>([\\s\\S]*?)</\\1>`,"g",);+ // `<skipped/>`, `<skipped message="..."/>` or `<skipped>...</skipped>`.+ // Non-global on purpose: `.test` would otherwise carry `lastIndex` between+ // testcases and start missing matches.+ const skippedRe = /<skipped\b/;for (const match of xml.matchAll(testcaseRe)) {const attrs = match[1];- const inner = match[2];- if (!inner) {+ const inner = match[2] ?? "";+ if (!inner && ignorePassingTests) {continue; // self-closing => passed}const problems = [...inner.matchAll(problemRe)];- if (problems.length === 0) {+ // Don't report a test to CI Conductor if the test has no failures and we are ignoring passing tests OR+ // the test has no failures and a skipped tag.+ if (problems.length === 0 && ( ignorePassingTests || skippedRe.test(inner))) {continue;}@@ -127,9 +136,7 @@ export function parseJunit(xml: string): NormalizedTest[] {file: file || null,message,stack: stack || undefined,- // JUnit only tells us a test failed/errored, never that it recovered,- // so everything is "failure".- status: "failure",+ status: problems.length === 0 && !ignorePassingTests ? "passed" : "failure",});}return tests;diff --git a/release/ci-conductor/src/report-backend.ts b/release/ci-conductor/src/report-backend.tsindex 215c272b4f0..b203036d259 100644--- a/release/ci-conductor/src/report-backend.ts+++ b/release/ci-conductor/src/report-backend.ts@@ -22,7 +22,10 @@ async function main(): Promise<void> {const testSuite =env.CI_CONDUCTOR_TEST_SUITE ||(env.DRIVERS ? `driver-${env.DRIVERS}` : "backend");- await reportTestFailures(normalizeBackendJunit(), testSuite);+ // If re-running tests we report failures AND passing tests so that ci-conductor can+ // determine flaky tests. We should not ignore passing tests in this case.+ const ignorePassingTests = env.IS_RERUN === "true" ? false : true;+ await reportTestFailures(normalizeBackendJunit(ignorePassingTests), testSuite);}main().catch((error) => {diff --git a/release/ci-conductor/src/report-frontend.ts b/release/ci-conductor/src/report-frontend.tsindex 5275f5cecf7..ce93dad0571 100644--- a/release/ci-conductor/src/report-frontend.ts+++ b/release/ci-conductor/src/report-frontend.ts@@ -20,7 +20,10 @@ async function main(): Promise<void> {// the shard number is deliberately NOT part of the suite, or a test would re-key in// ci-conductor whenever sharding changes. Falls back to a generic `frontend`.const testSuite = env.CI_CONDUCTOR_TEST_SUITE || "frontend";- await reportTestFailures(normalizeFrontendJunit(), testSuite);+ // If re-running tests we report failures AND passing tests so that ci-conductor can+ // determine flaky tests. We should not ignore passing tests in this case.+ const ignorePassingTests = env.IS_RERUN === "true" ? false : true;+ await reportTestFailures(normalizeFrontendJunit(ignorePassingTests), testSuite);}main().catch((error) => {