Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

report all test results on rerun to CI Conductor (#80326)

Chris Vandoodewaarde30fb0cec1e9
diff --git a/.github/actions/report-failures-to-ci-conductor/action.yml b/.github/actions/report-failures-to-ci-conductor/action.yml
index 6a0c5245899..b61c2d81a41 100644
--- a/.github/actions/report-failures-to-ci-conductor/action.yml
+++ b/.github/actions/report-failures-to-ci-conductor/action.yml
@@ -25,6 +25,7 @@ runs:
TARGET_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
CI_CONDUCTOR_TEST_SUITE: ${{ inputs.test-suite }}
ADAPTER: ${{ inputs.adapter }}
+ IS_RERUN: ${{ github.run_attempt != '1' }}
# Keep the path constrained to known entrypoints.
run: |
case "$ADAPTER" in
diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.yml
index 341077a739a..a34f1feb08d 100644
--- a/.github/actions/test-driver/action.yml
+++ b/.github/actions/test-driver/action.yml
@@ -114,7 +114,7 @@ runs:
continue-on-error: true
- name: Report failed tests to ci-conductor
- if: ${{ !cancelled() && steps.run-test.outcome == 'failure' }}
+ if: ${{ !cancelled() && ( steps.run-test.outcome == 'failure' || github.run_attempt != '1' ) }}
# This step should NEVER fail the job!
continue-on-error: true
uses: ./.github/actions/report-failures-to-ci-conductor
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 52039a31896..7cfb922c9bf 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -232,7 +232,7 @@ jobs:
continue-on-error: true
- name: Report failed tests to ci-conductor
- if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}
+ if: ${{ !cancelled() && ( steps.run-java-tests.outcome == 'failure' || github.run_attempt != '1' ) }}
# This step should NEVER fail the job!
continue-on-error: true
uses: ./.github/actions/report-failures-to-ci-conductor
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
index 552b43b6c9a..570cf78b8a8 100644
--- a/.github/workflows/frontend.yml
+++ b/.github/workflows/frontend.yml
@@ -116,7 +116,7 @@ jobs:
# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.
continue-on-error: true
- name: Report failed tests to ci-conductor
- if: ${{ !cancelled() && steps.fe-tests.outcome == 'failure' }}
+ if: ${{ !cancelled() && ( steps.fe-tests.outcome == 'failure' || github.run_attempt != '1' ) }}
# This step should NEVER fail the job!
continue-on-error: true
uses: ./.github/actions/report-failures-to-ci-conductor
diff --git a/release/ci-conductor/src/adapters/backend.ts b/release/ci-conductor/src/adapters/backend.ts
index 284fce0cd6d..12f27c0549c 100644
--- a/release/ci-conductor/src/adapters/backend.ts
+++ b/release/ci-conductor/src/adapters/backend.ts
@@ -33,22 +33,23 @@ const selectHawkJunit = (entries: string[]): string[] =>
* normalized shape).
*/
export function normalizeBackendJunit(
+ ignorePassingTests: boolean = true,
dir: string = JUNIT_DIR,
): NormalizedTest[] {
const files = findJunitFiles(dir, selectHawkJunit);
const failures = files.flatMap((file) => {
try {
- return parseJunit(readFileSync(file, "utf8"));
+ return parseJunit(readFileSync(file, "utf8"), ignorePassingTests);
} catch (error) {
console.error(`[ci-conductor] failed to read ${file}`, error);
return [];
}
});
log(
- `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} failing test(s)`,
+ `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} ${ignorePassingTests ? "failing " : ""}test(s)`,
);
for (const test of failures) {
- log(` failing: ${test.path || "(no namespace)"} / ${test.name}`);
+ log(` ${test.status ?? 'failure'}: ${test.path || "(no namespace)"} / ${test.name}`);
}
return failures;
}
diff --git a/release/ci-conductor/src/adapters/frontend.ts b/release/ci-conductor/src/adapters/frontend.ts
index 4d7b8efcda0..b8a3a96354f 100644
--- a/release/ci-conductor/src/adapters/frontend.ts
+++ b/release/ci-conductor/src/adapters/frontend.ts
@@ -46,23 +46,24 @@ function selectJestJunit(name: string): (entries: string[]) => string[] {
* (jest-junit XML → the shared normalized shape).
*/
export function normalizeFrontendJunit(
+ ignorePassingTests: boolean = true,
dir: string = JUNIT_DIR,
name: string = JUNIT_NAME,
): NormalizedTest[] {
const files = findJunitFiles(dir, selectJestJunit(name));
const failures = files.flatMap((file) => {
try {
- return parseJunit(readFileSync(file, "utf8"));
+ return parseJunit(readFileSync(file, "utf8"), ignorePassingTests);
} catch (error) {
console.error(`[ci-conductor] failed to read ${file}`, error);
return [];
}
});
log(
- `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} failing test(s)`,
+ `scanned ${dir}: ${files.length} JUnit file(s), ${failures.length} ${ignorePassingTests ? "failing " : ""}test(s)`,
);
for (const test of failures) {
- log(` failing: ${test.path || "(no describe path)"} / ${test.name}`);
+ log(` ${test.status ?? 'failure'}: ${test.path || "(no describe path)"} / ${test.name}`);
}
return failures;
}
diff --git a/release/ci-conductor/src/junit.test.ts b/release/ci-conductor/src/junit.test.ts
index 8c8b0698b8c..b3bb0339fb6 100644
--- a/release/ci-conductor/src/junit.test.ts
+++ b/release/ci-conductor/src/junit.test.ts
@@ -148,6 +148,48 @@ const JEST_WITH_FILE = `<?xml version="1.0" encoding="UTF-8"?>
</testsuite>
</testsuites>`;
+// A jest-junit document as it looks on a *rerun*: mostly passes (self-closing
+// testcases) with one failure. This is the shape `ignorePassingTests: false`
+// exists for — ci-conductor needs the passes to tell a flake (failed, then
+// passed on rerun) from a genuine failure.
+const JEST_MOSTLY_PASSING = `<?xml version="1.0" encoding="UTF-8"?>
+<testsuites name="jest tests" tests="3" failures="1">
+<testsuite name="Button" failures="1" tests="3">
+<testcase classname="Button renders" name="Button renders" time="0.01" file="frontend/src/metabase/components/Button/Button.unit.spec.tsx"/>
+<testcase classname="Button is disabled" name="Button is disabled" time="0.02" file="frontend/src/metabase/components/Button/Button.unit.spec.tsx"/>
+<testcase classname="Button explodes" name="Button explodes" time="0.03" file="frontend/src/metabase/components/Button/Button.unit.spec.tsx">
+<failure message="expect(received).toBe(expected)">Error: expect(received).toBe(expected)</failure>
+</testcase>
+</testsuite>
+</testsuites>`;
+
+// A testcase that has a body but no <failure>/<error> — jest-junit emits this
+// shape when a passing test wrote to stdout. It passed, same as a self-closing
+// one.
+const PASSING_WITH_BODY = `<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="metabase.foo-test" tests="1" failures="0">
+<testcase classname="metabase.foo-test" name="chatty-test" time="0.01">
+<system-out>some log output</system-out>
+</testcase>
+</testsuite>`;
+
+// jest marks a skipped test (`it.skip`/`xit`, or a suite it never ran) with a
+// <skipped/> child; a producer may also write it with a `message` attribute or a
+// body. A real run mixes skips in among the passes and failures.
+const SKIPPED = `<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="Button" tests="4" failures="1" skipped="2">
+<testcase classname="Button is todo" name="Button is todo" time="0">
+<skipped/>
+</testcase>
+<testcase classname="Button is deferred" name="Button is deferred" time="0">
+<skipped message="pending"/>
+</testcase>
+<testcase classname="Button renders" name="Button renders" time="0.01"/>
+<testcase classname="Button explodes" name="Button explodes" time="0.02">
+<failure message="boom">Error: boom</failure>
+</testcase>
+</testsuite>`;
+
describe("parseJunit", () => {
it("prefers the failure `message` attribute over the body", () => {
const [test] = parseJunit(FAILURE_WITH_MESSAGE);
@@ -242,6 +284,11 @@ describe("parseJunit", () => {
expect(parseJunit("<not-valid")).toEqual([]);
});
+ it("drops skipped tests, keeping only the failure", () => {
+ const tests = parseJunit(SKIPPED);
+ expect(tests.map((t) => t.name)).toEqual(["Button explodes"]);
+ });
+
it("returns [] for a suite with only passing tests", () => {
const xml = `<testsuite name="metabase.ok-test" tests="1" failures="0">
<testcase classname="metabase.ok-test" name="ok" time="0.01"/>
@@ -249,3 +296,82 @@ describe("parseJunit", () => {
expect(parseJunit(xml)).toEqual([]);
});
});
+
+describe("parseJunit with ignorePassingTests: false", () => {
+ it("emits self-closing (passing) testcases with status `passed`", () => {
+ const tests = parseJunit(PASS_AND_ERROR, false);
+ expect(tests).toHaveLength(2);
+ const passed = tests.find((t) => t.name === "passing-test");
+ expect(passed).toBeDefined();
+ expect(passed!.status).toBe("passed");
+ expect(passed!.path).toBe("metabase.foo-test");
+ // Nothing went wrong, so there's no message or stack to carry.
+ expect(passed!.message).toBeNull();
+ expect(passed!.stack).toBeUndefined();
+ });
+
+ it("still reports failures as failures alongside the passes", () => {
+ const tests = parseJunit(PASS_AND_ERROR, false);
+ const errored = tests.find((t) => t.name === "erroring-test");
+ expect(errored).toBeDefined();
+ expect(errored!.status).toBe("failure");
+ expect(errored!.stack).toContain("java.lang.RuntimeException: boom");
+ });
+
+ it("reports every test in a mostly-passing jest rerun, keeping `file`", () => {
+ const tests = parseJunit(JEST_MOSTLY_PASSING, false);
+ expect(
+ tests.map((t) => [t.name, t.status]).sort((a, b) => a[0]!.localeCompare(b[0]!)),
+ ).toEqual([
+ ["Button explodes", "failure"],
+ ["Button is disabled", "passed"],
+ ["Button renders", "passed"],
+ ]);
+ expect(
+ tests.every(
+ (t) =>
+ t.file ===
+ "frontend/src/metabase/components/Button/Button.unit.spec.tsx",
+ ),
+ ).toBe(true);
+ });
+
+ it("treats a testcase with a body but no <failure>/<error> as passed", () => {
+ const tests = parseJunit(PASSING_WITH_BODY, false);
+ expect(tests).toHaveLength(1);
+ expect(tests[0].status).toBe("passed");
+ expect(parseJunit(PASSING_WITH_BODY)).toEqual([]);
+ });
+
+ it("reports a suite of only passing tests instead of []", () => {
+ const xml = `<testsuite name="metabase.ok-test" tests="1" failures="0">
+<testcase classname="metabase.ok-test" name="ok" time="0.01"/>
+</testsuite>`;
+ expect(parseJunit(xml, false)).toEqual([
+ {
+ name: "ok",
+ path: "metabase.ok-test",
+ file: null,
+ message: null,
+ stack: undefined,
+ status: "passed",
+ },
+ ]);
+ });
+
+ it("still returns [] for malformed XML without throwing", () => {
+ expect(parseJunit("<not-valid", false)).toEqual([]);
+ });
+
+ it("drops skipped tests while keeping the passes and failures around them", () => {
+ const tests = parseJunit(SKIPPED, false);
+ expect(
+ tests
+ .map((t) => [t.name, t.status])
+ .sort((a, b) => a[0]!.localeCompare(b[0]!)),
+ ).toEqual([
+ ["Button explodes", "failure"],
+ ["Button renders", "passed"],
+ ]);
+ });
+});
diff --git a/release/ci-conductor/src/junit.ts b/release/ci-conductor/src/junit.ts
index 3ea578973a5..4e1d699f703 100644
--- a/release/ci-conductor/src/junit.ts
+++ b/release/ci-conductor/src/junit.ts
@@ -56,15 +56,18 @@ function elementBody(inner: string): string {
}
/**
- * Parse one JUnit XML document into normalized entries — one per `<testcase>`
- * that carries a `<failure>` or `<error>`. Multiple problems in a single
- * testcase are joined into one `stack`. Passing (self-closing or problem-free)
- * testcases are skipped. Never throws.
+ * Parse one JUnit XML document into normalized entries. Multiple problems in a
+ * single `<testcase>` are joined into one `stack`. Never throws.
+ *
+ * By default only testcases carrying a `<failure>` or `<error>` are emitted.
+ * Pass `ignorePassingTests: false` on a rerun to also emit the passes (as
+ * `status: "passed"`), which is what lets ci-conductor tell a recovered flake
+ * from a still-broken test. Either way, `<skipped/>` testcases are dropped.
*
* Pure (string → normalized), so it's the unit-tested core that each suite's
* adapter wraps with its own file discovery / labeling.
*/
-export function parseJunit(xml: string): NormalizedTest[] {
+export function parseJunit(xml: string, ignorePassingTests: boolean = true): NormalizedTest[] {
try {
const tests: NormalizedTest[] = [];
// <testcase ...>...</testcase> (failing) or <testcase .../> (passing,
@@ -85,15 +88,21 @@ export function parseJunit(xml: string): NormalizedTest[] {
`<(failure|error)\\b(${ATTRS})>([\\s\\S]*?)</\\1>`,
"g",
);
+ // `<skipped/>`, `<skipped message="..."/>` or `<skipped>...</skipped>`.
+ // Non-global on purpose: `.test` would otherwise carry `lastIndex` between
+ // testcases and start missing matches.
+ const skippedRe = /<skipped\b/;
for (const match of xml.matchAll(testcaseRe)) {
const attrs = match[1];
- const inner = match[2];
- if (!inner) {
+ const inner = match[2] ?? "";
+ if (!inner && ignorePassingTests) {
continue; // self-closing => passed
}
const problems = [...inner.matchAll(problemRe)];
- if (problems.length === 0) {
+ // Don't report a test to CI Conductor if the test has no failures and we are ignoring passing tests OR
+ // the test has no failures and a skipped tag.
+ if (problems.length === 0 && ( ignorePassingTests || skippedRe.test(inner))) {
continue;
}
@@ -127,9 +136,7 @@ export function parseJunit(xml: string): NormalizedTest[] {
file: file || null,
message,
stack: stack || undefined,
- // JUnit only tells us a test failed/errored, never that it recovered,
- // so everything is "failure".
- status: "failure",
+ status: problems.length === 0 && !ignorePassingTests ? "passed" : "failure",
});
}
return tests;
diff --git a/release/ci-conductor/src/report-backend.ts b/release/ci-conductor/src/report-backend.ts
index 215c272b4f0..b203036d259 100644
--- a/release/ci-conductor/src/report-backend.ts
+++ b/release/ci-conductor/src/report-backend.ts
@@ -22,7 +22,10 @@ async function main(): Promise<void> {
const testSuite =
env.CI_CONDUCTOR_TEST_SUITE ||
(env.DRIVERS ? `driver-${env.DRIVERS}` : "backend");
- await reportTestFailures(normalizeBackendJunit(), testSuite);
+ // If re-running tests we report failures AND passing tests so that ci-conductor can
+ // determine flaky tests. We should not ignore passing tests in this case.
+ const ignorePassingTests = env.IS_RERUN === "true" ? false : true;
+ await reportTestFailures(normalizeBackendJunit(ignorePassingTests), testSuite);
}
main().catch((error) => {
diff --git a/release/ci-conductor/src/report-frontend.ts b/release/ci-conductor/src/report-frontend.ts
index 5275f5cecf7..ce93dad0571 100644
--- a/release/ci-conductor/src/report-frontend.ts
+++ b/release/ci-conductor/src/report-frontend.ts
@@ -20,7 +20,10 @@ async function main(): Promise<void> {
// the shard number is deliberately NOT part of the suite, or a test would re-key in
// ci-conductor whenever sharding changes. Falls back to a generic `frontend`.
const testSuite = env.CI_CONDUCTOR_TEST_SUITE || "frontend";
- await reportTestFailures(normalizeFrontendJunit(), testSuite);
+ // If re-running tests we report failures AND passing tests so that ci-conductor can
+ // determine flaky tests. We should not ignore passing tests in this case.
+ const ignorePassingTests = env.IS_RERUN === "true" ? false : true;
+ await reportTestFailures(normalizeFrontendJunit(ignorePassingTests), testSuite);
}
main().catch((error) => {