Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Send cypress test failures info to ci conductor (#74872)

Nemanja Glumac3599d9f88513
diff --git a/.github/actions/resolve-job-id/action.yml b/.github/actions/resolve-job-id/action.yml
new file mode 100644
index 00000000000..25cc81b64cf
--- /dev/null
+++ b/.github/actions/resolve-job-id/action.yml
@@ -0,0 +1,48 @@
+name: Resolve current job id
+description: >
+ Resolve this job's GitHub workflow_jobs.id via the Actions API and export it
+ as JOB_ID (and JOB_HTML_URL) to $GITHUB_ENV, so any later step can read the
+ numeric job id without making its own API call. Best-effort: warns and
+ proceeds on failure so it can never break the run.
+
+runs:
+ using: composite
+ steps:
+ - name: Resolve JOB_ID
+ shell: bash
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RUNNER_NAME: ${{ runner.name }}
+ run: |
+ # GitHub doesn't expose the current job's numeric id, so we look it up
+ # from the Actions API. A runner runs exactly one job at a time, so
+ # filtering by runner_name within (run_id, run_attempt) is unique.
+ # max_by(.started_at) handles the rare case of a runner sequentially
+ # running multiple jobs in the same attempt — we want the latest.
+ set +e
+ api_url="/repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100"
+ jq_filter="[.jobs[] | select(.runner_name == \"$RUNNER_NAME\")] | max_by(.started_at) | \"\(.id) \(.html_url)\""
+
+ # The Actions jobs API is eventually consistent: after a runner picks
+ # up a job, it can take a few seconds for runner_name to appear in the
+ # response. Retry briefly to absorb that.
+ out=""
+ for attempt in 1 2 3 4 5; do
+ out=$(gh api "$api_url" --jq "$jq_filter" 2>/dev/null)
+ if [ -n "$out" ] && [ "$out" != "null null" ]; then
+ break
+ fi
+ out=""
+ echo "Attempt $attempt: job not yet visible in Actions API; retrying..."
+ sleep 3
+ done
+
+ if [ -n "$out" ]; then
+ read -r job_id job_url <<<"$out"
+ echo "JOB_ID=$job_id" >> "$GITHUB_ENV"
+ echo "JOB_HTML_URL=$job_url" >> "$GITHUB_ENV"
+ echo "Resolved JOB_ID=$job_id (runner=$RUNNER_NAME)"
+ else
+ echo "::warning::Could not resolve current job id after 5 attempts (runner=$RUNNER_NAME)"
+ fi
+ exit 0
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index 8e487d27328..6050c352e63 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -37,6 +37,19 @@ env:
HASH: ${{ github.event.pull_request.head.sha || github.sha }}-${{ github.run_attempt }}
PR_NUMBER: ${{ github.event.pull_request.number || '' }}
JOB_NAME: ${{ inputs.name }}
+ # ci-conductor failed-tests reporting (DEV-1999). Secret holds the webhooks
+ # base URL (".../webhooks"); the reporter appends the endpoint. Callers pass
+ # `secrets: inherit`; reporting no-ops when it's unset.
+ CI_CONDUCTOR_WEBHOOK_URL: ${{ secrets.CI_CONDUCTOR_WEBHOOK_URL }}
+ # Shared secret sent as the `x-internal-secret` header to authenticate.
+ CI_CONDUCTOR_WEBHOOK_SECRET: ${{ secrets.CI_CONDUCTOR_WEBHOOK_SECRET }}
+ REPO_ID: ${{ github.event.repository.id }}
+ # Commit + PR target branch under test, so conductor can tie failures to the
+ # right code. On PRs the ambient GITHUB_SHA is the synthetic merge commit and
+ # GITHUB_BASE_REF the target, so we pass the PR head sha / base ref explicitly
+ # (falling back to push-event values). The reporter reads these. DEV-1999.
+ COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+ TARGET_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
jobs:
e2e-tests:
@@ -112,6 +125,15 @@ jobs:
- name: Run Metabase
run: node e2e/runner/run_cypress_ci.js start
+ # Export this job's numeric GitHub job id as $JOB_ID for the Cypress runs
+ # below (ci-conductor failed-tests reporting reads it in after:spec).
+ # Snapshot generation is itself a Cypress run that fires after:spec, so
+ # this must precede it. Placed late, not after checkout: the Actions jobs
+ # API is eventually consistent, so the longer the job has been alive the
+ # more reliably the lookup resolves. JOB_ID exported here persists to the
+ # steps below. DEV-1999.
+ - uses: ./.github/actions/resolve-job-id
+
- name: Generate database snapshots
env:
CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}
diff --git a/e2e/support/ci_conductor.ts b/e2e/support/ci_conductor.ts
new file mode 100644
index 00000000000..1f6797d4c0b
--- /dev/null
+++ b/e2e/support/ci_conductor.ts
@@ -0,0 +1,240 @@
+import fetch from "node-fetch"; // must be node-fetch v2 because it's non-esm
+
+// `Cypress` and `CypressCommandLine` are global namespaces provided by the
+// "cypress" types (see e2e/tsconfig.json), so they're referenced without import.
+
+/**
+ * Reports Cypress test failures to the ci-conductor service, mid-run, from the
+ * `after:spec` Node hook (see e2e/support/config.js). Sending per-spec — rather
+ * than waiting for the whole job to finish — lets failures be tied back to the
+ * CI run (and through it the PR / release branch) before the job completes.
+ *
+ * The webhooks base URL is only supplied via env in CI, so local runs no-op.
+ *
+ * See DEV-1999.
+ */
+
+const {
+ CI_CONDUCTOR_WEBHOOK_URL,
+ CI_CONDUCTOR_WEBHOOK_SECRET,
+ CI_CONDUCTOR_DRY_RUN,
+ REPO_ID,
+ GITHUB_RUN_ID,
+ GITHUB_RUN_ATTEMPT,
+ JOB_ID,
+ // The commit and PR target branch under test. On PRs the ambient GITHUB_SHA is
+ // a synthetic merge commit and GITHUB_BASE_REF the target, so e2e-test.yml sets
+ // COMMIT_SHA/TARGET_BRANCH to the PR's head sha / base ref; we fall back to the
+ // ambient vars (push runs, local) when they're unset. See DEV-1999.
+ COMMIT_SHA,
+ GITHUB_SHA,
+ TARGET_BRANCH,
+ GITHUB_BASE_REF,
+} = process.env;
+
+// When set, the payload is logged instead of POSTed — used to validate env
+// resolution and payload shape in CI before sending real data. See DEV-1999.
+const isDryRun = CI_CONDUCTOR_DRY_RUN === "true";
+
+/** Matches the `tests[]` shape consumed by ci-conductor's `ingestFailedTests`. */
+type ConductorTest = {
+ name: string;
+ /** The test's suite path (the joined `describe` titles), formerly `class`. */
+ path?: string;
+ file?: string;
+ duration?: number;
+ /** Raw per-attempt shape from Cypress, e.g. [{state:"failed"},{state:"passed"}]. */
+ attempts?: { state: string }[];
+ /**
+ * "failure" when every attempt failed (broken), "flake" when it failed then
+ * passed on retry, "passed" when every attempt passed. Passes are only
+ * reported on re-runs (run attempt > 1). Derived from `attempts`.
+ */
+ status?: "failure" | "flake" | "passed";
+ /**
+ * Cypress' final `displayError` blob for the test. Null for flaky tests
+ * (Cypress drops it when the final attempt passes); only broken tests carry a
+ * value. Conductor schema isn't final — fields it doesn't store are ignored.
+ */
+ message?: string | null;
+};
+
+/**
+ * Classify a test from its attempts: "passed" if every attempt passed,
+ * "failure" if every attempt failed, "flake" if it failed at least once but
+ * ultimately passed on retry. Callers only pass tests that ran (a non-empty
+ * attempts array of passes and/or fails), never pending/skipped.
+ */
+function classifyStatus(
+ attempts: { state: string }[],
+): "failure" | "flake" | "passed" {
+ if (attempts.every((attempt) => attempt.state === "passed")) {
+ return "passed";
+ }
+ const allFailed = attempts.every((attempt) => attempt.state === "failed");
+ return allFailed ? "failure" : "flake";
+}
+
+/**
+ * GitHub doesn't expose the current job's numeric `workflow_jobs.id` to a
+ * running job, so the `./.github/actions/resolve-job-id` composite action
+ * resolves it once at job start and exports it as `JOB_ID`. We just read that
+ * env here. Falls back to null when the env isn't set (the column is
+ * nullable). See DEV-1999.
+ */
+function getJobId(): number | null {
+ return toNumber(JOB_ID);
+}
+
+/** Parse a numeric env var, treating missing/blank/non-numeric as null. */
+function toNumber(value: string | undefined): number | null {
+ if (value == null || value === "") {
+ return null;
+ }
+ const parsed = Number(value);
+ return Number.isFinite(parsed) ? parsed : null;
+}
+
+/**
+ * Pull the reportable tests out of a single spec's run results. We always
+ * include any test that had at least one failed attempt — so both "broken"
+ * (every attempt failed) and "flaky" (failed at least once, passed on retry)
+ * are reported. On a re-run (GITHUB_RUN_ATTEMPT > 1) we additionally include
+ * tests that passed, so conductor can see a previously-failing test recover.
+ * Pending/skipped tests are never reported. Each row carries a derived `status`
+ * ("failure" | "flake" | "passed"); see `classifyStatus`.
+ *
+ * Cypress only populates `displayError` for the *final* state, so flaky tests
+ * arrive with `message: null` — we know *that* they flaked from `attempts`,
+ * but not *why*. Broken tests carry the full error blob.
+ *
+ * If the spec crashed before any tests ran (e.g. a compile/import error),
+ * Cypress reports no tests but sets `results.error`; we surface that as a
+ * single synthetic entry so the failure isn't lost.
+ */
+export function extractFailedTests(
+ spec: Cypress.Spec,
+ results: CypressCommandLine.RunResult,
+): ConductorTest[] {
+ const file = spec?.relative;
+
+ // On a re-run we also report passing tests (not just failures) so conductor
+ // can see a previously-failing test recover. Read at call time rather than the
+ // module-level destructure so it's controllable per-call in tests.
+ const isRerun = (toNumber(process.env.GITHUB_RUN_ATTEMPT) ?? 0) > 1;
+
+ const tests = (results?.tests ?? [])
+ .filter((test) => {
+ const attempts = test.attempts ?? [];
+ const failed = attempts.some((attempt) => attempt.state === "failed");
+ // A test "passed" only if it ran and every attempt passed — this excludes
+ // pending/skipped tests, which we never report.
+ const passed =
+ attempts.length > 0 &&
+ attempts.every((attempt) => attempt.state === "passed");
+ return failed || (isRerun && passed);
+ })
+ .map((test) => {
+ const titlePath = test.title ?? [];
+ const name = titlePath[titlePath.length - 1] ?? "(unknown test)";
+ const suite = titlePath.slice(0, -1).join(" > ");
+ const attempts = test.attempts ?? [];
+ return {
+ name,
+ path: suite || undefined,
+ file,
+ duration: test.duration,
+ attempts,
+ status: classifyStatus(attempts),
+ message: test.displayError ?? null,
+ };
+ });
+
+ if (tests.length === 0 && results?.error) {
+ return [
+ {
+ name: spec?.name ?? "(spec failed to run)",
+ file,
+ attempts: [{ state: "failed" }],
+ status: "failure",
+ message: results.error,
+ },
+ ];
+ }
+
+ return tests;
+}
+
+/**
+ * Report the given failures to ci-conductor. In dry-run mode the payload is
+ * logged and nothing is sent. Otherwise it's POSTed, no-opping when the webhook
+ * URL isn't configured (local runs, PRs without the secret). Never throws —
+ * reporting must not break a test run — so all errors are logged and swallowed.
+ */
+export async function reportFailedTestsToConductor(
+ tests: ConductorTest[],
+): Promise<void> {
+ if (tests.length === 0 || (!CI_CONDUCTOR_WEBHOOK_URL && !isDryRun)) {
+ return;
+ }
+
+ // Everything below is wrapped so the reporter can never throw into the test
+ // run, regardless of payload contents or network behavior.
+ try {
+ const body = {
+ repo_id: toNumber(REPO_ID),
+ run_id: toNumber(GITHUB_RUN_ID),
+ attempt: toNumber(GITHUB_RUN_ATTEMPT),
+ job_id: getJobId(),
+ test_suite: "e2e",
+ // PR head sha / target branch when set by e2e-test.yml, else the ambient
+ // (push/local) values. Empty strings collapse to null.
+ sha: COMMIT_SHA || GITHUB_SHA || null,
+ target_branch: TARGET_BRANCH || GITHUB_BASE_REF || null,
+ // The retry ceiling so conductor can interpret per-test `attempts`.
+ // CYPRESS_RETRIES isn't set in CI by default; e2e/support/config.js
+ // surfaces the resolved Cypress config value into this env at startup.
+ retries: toNumber(process.env.CYPRESS_RETRIES),
+ tests,
+ };
+
+ if (isDryRun) {
+ console.log(
+ `[ci-conductor] (dry run) would POST ${tests.length} failure(s):`,
+ JSON.stringify(body),
+ );
+ return;
+ }
+
+ if (!CI_CONDUCTOR_WEBHOOK_URL) {
+ // already excluded by the guard above; this also narrows the type
+ return;
+ }
+
+ // CI_CONDUCTOR_WEBHOOK_URL holds the webhooks *base* (e.g. ".../webhooks");
+ // we append the specific endpoint so the same secret can serve others later.
+ const endpoint = `${CI_CONDUCTOR_WEBHOOK_URL.replace(/\/+$/, "")}/failed-tests`;
+
+ // ci-conductor authenticates this endpoint via the x-internal-secret header.
+ const headers: Record<string, string> = {
+ "Content-Type": "application/json",
+ };
+ if (CI_CONDUCTOR_WEBHOOK_SECRET) {
+ headers["x-internal-secret"] = CI_CONDUCTOR_WEBHOOK_SECRET;
+ }
+
+ const response = await fetch(endpoint, {
+ method: "POST",
+ headers,
+ body: JSON.stringify(body),
+ });
+
+ if (!response.ok) {
+ console.error(
+ `[ci-conductor] failed-tests POST returned ${response.status} ${response.statusText}`,
+ );
+ }
+ } catch (error) {
+ console.error("[ci-conductor] failed to POST failed-tests", error);
+ }
+}
diff --git a/e2e/support/ci_conductor.unit.spec.ts b/e2e/support/ci_conductor.unit.spec.ts
new file mode 100644
index 00000000000..c6dce2ddbfb
--- /dev/null
+++ b/e2e/support/ci_conductor.unit.spec.ts
@@ -0,0 +1,496 @@
+import { extractFailedTests } from "./ci_conductor";
+
+// jest.mock is hoisted; the mocked default must be referenced via a `mock`-
+// prefixed variable. node-fetch is unused by extractFailedTests, so mocking it
+// here is harmless for those tests.
+const mockFetch = jest.fn();
+jest.mock("node-fetch", () => ({
+ __esModule: true,
+ default: (...args: unknown[]) => mockFetch(...args),
+}));
+
+// ci_conductor reads some env at import time (top-level destructure) and some
+// at call time (e.g. CYPRESS_RETRIES), so loadConductor applies the env BEFORE
+// re-importing and leaves it applied until afterEach restores it.
+const envBackups: Array<{ key: string; previous: string | undefined }> = [];
+
+afterEach(() => {
+ while (envBackups.length) {
+ const { key, previous } = envBackups.pop()!;
+ if (previous === undefined) {
+ delete process.env[key];
+ } else {
+ process.env[key] = previous;
+ }
+ }
+});
+
+const loadConductor = async (env: Record<string, string | undefined>) => {
+ jest.resetModules();
+ for (const [key, value] of Object.entries(env)) {
+ envBackups.push({ key, previous: process.env[key] });
+ if (value === undefined) {
+ delete process.env[key];
+ } else {
+ process.env[key] = value;
+ }
+ }
+ return await import("./ci_conductor");
+};
+
+// Minimal builders for the bits of Cypress' after:spec payload we read.
+const spec = {
+ name: "foo.cy.spec.ts",
+ relative: "e2e/test/scenarios/foo/foo.cy.spec.ts",
+} as Cypress.Spec;
+
+const makeResults = (
+ overrides: Partial<CypressCommandLine.RunResult>,
+): CypressCommandLine.RunResult =>
+ ({
+ error: null,
+ tests: [],
+ stats: { failures: 0 },
+ ...overrides,
+ }) as unknown as CypressCommandLine.RunResult;
+
+// Builds a TestResult-shaped object. The final `state` is derived from
+// `attemptStates` so callers can describe healthy/flaky/broken naturally.
+const test = (
+ title: string[],
+ attemptStates: string[],
+ displayError: string | null = null,
+ duration: number = 100,
+) => ({
+ title,
+ state: attemptStates[attemptStates.length - 1],
+ displayError,
+ duration,
+ attempts: attemptStates.map((state) => ({ state })),
+});
+
+describe("extractFailedTests", () => {
+ // Pin the default to "first attempt" so these tests are deterministic even
+ // when the CI job running them is itself a re-run (GITHUB_RUN_ATTEMPT > 1).
+ // The global afterEach restores the original via envBackups. Re-run cases
+ // override process.env.GITHUB_RUN_ATTEMPT in the test body.
+ beforeEach(() => {
+ envBackups.push({
+ key: "GITHUB_RUN_ATTEMPT",
+ previous: process.env.GITHUB_RUN_ATTEMPT,
+ });
+ delete process.env.GITHUB_RUN_ATTEMPT;
+ });
+
+ it("reports broken tests with the full displayError as message", () => {
+ const displayError =
+ "AssertionError: expected true to be false\n at Context.eval (foo.cy.spec.ts:42)";
+
+ const results = makeResults({
+ tests: [
+ test(["Dashboard", "filters", "applies a filter"], ["passed"]),
+ test(
+ ["Dashboard", "filters", "shows an error"],
+ ["failed", "failed"],
+ displayError,
+ ),
+ ],
+ });
+
+ expect(extractFailedTests(spec, results)).toEqual([
+ {
+ name: "shows an error",
+ path: "Dashboard > filters",
+ file: "e2e/test/scenarios/foo/foo.cy.spec.ts",
+ duration: 100,
+ attempts: [{ state: "failed" }, { state: "failed" }],
+ status: "failure",
+ message: displayError,
+ },
+ ]);
+ });
+
+ it("includes flaky tests too (final passed but ≥1 failed attempt), with null message", () => {
+ const results = makeResults({
+ tests: [
+ test(["a", "healthy"], ["passed"]),
+ test(["a", "flaky"], ["failed", "passed"]),
+ ],
+ });
+
+ const failures = extractFailedTests(spec, results);
+ expect(failures).toHaveLength(1);
+ expect(failures[0]).toMatchObject({
+ name: "flaky",
+ attempts: [{ state: "failed" }, { state: "passed" }],
+ status: "flake",
+ message: null,
+ });
+ });
+
+ it("excludes healthy, pending, and skipped tests", () => {
+ const results = makeResults({
+ tests: [
+ test(["a", "passed test"], ["passed"]),
+ test(["a", "pending test"], ["pending"]),
+ test(["a", "skipped test"], ["skipped"]),
+ ],
+ });
+
+ expect(extractFailedTests(spec, results)).toEqual([]);
+ });
+
+ it("returns an empty array when nothing has a failed attempt", () => {
+ expect(extractFailedTests(spec, makeResults({}))).toEqual([]);
+ });
+
+ it("omits `path` for a top-level test with no suite", () => {
+ const results = makeResults({
+ tests: [test(["lonely test"], ["failed"], "boom")],
+ });
+
+ const [failure] = extractFailedTests(spec, results);
+ expect(failure).toMatchObject({ name: "lonely test", status: "failure" });
+ expect(failure.path).toBeUndefined();
+ });
+
+ it("sends message: null when a failed test has no displayError", () => {
+ const results = makeResults({
+ tests: [test(["a", "no error string"], ["failed"], null)],
+ });
+
+ expect(extractFailedTests(spec, results)[0]).toMatchObject({
+ name: "no error string",
+ path: "a",
+ message: null,
+ });
+ });
+
+ it("falls back to a synthetic entry when the spec crashes before any test runs", () => {
+ const results = makeResults({
+ error: "SyntaxError: Unexpected token\n at compile",
+ tests: [],
+ });
+
+ expect(extractFailedTests(spec, results)).toEqual([
+ {
+ name: "foo.cy.spec.ts",
+ file: "e2e/test/scenarios/foo/foo.cy.spec.ts",
+ attempts: [{ state: "failed" }],
+ status: "failure",
+ message: "SyntaxError: Unexpected token\n at compile",
+ },
+ ]);
+ });
+
+ it("prefers real failed tests over the spec-level error fallback", () => {
+ const results = makeResults({
+ error: "some run-level error",
+ tests: [test(["a", "real failure"], ["failed"], "boom")],
+ });
+
+ const failures = extractFailedTests(spec, results);
+ expect(failures).toHaveLength(1);
+ expect(failures[0].name).toBe("real failure");
+ });
+
+ it("tolerates a missing tests array", () => {
+ expect(extractFailedTests(spec, makeResults({ tests: undefined }))).toEqual(
+ [],
+ );
+ });
+
+ it("does not report passing tests on the first attempt", () => {
+ process.env.GITHUB_RUN_ATTEMPT = "1";
+ const results = makeResults({
+ tests: [test(["a", "passing"], ["passed"])],
+ });
+
+ expect(extractFailedTests(spec, results)).toEqual([]);
+ });
+
+ it("on a re-run (attempt > 1) also reports passing tests with status passed", () => {
+ process.env.GITHUB_RUN_ATTEMPT = "2";
+ const results = makeResults({
+ tests: [
+ test(["a", "now passing"], ["passed"]),
+ test(["a", "still broken"], ["failed", "failed"], "boom"),
+ ],
+ });
+
+ const reported = extractFailedTests(spec, results);
+ expect(reported).toHaveLength(2);
+ expect(reported).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ name: "now passing",
+ status: "passed",
+ message: null,
+ }),
+ expect.objectContaining({ name: "still broken", status: "failure" }),
+ ]),
+ );
+ });
+
+ it("still excludes pending and skipped tests on a re-run", () => {
+ process.env.GITHUB_RUN_ATTEMPT = "3";
+ const results = makeResults({
+ tests: [
+ test(["a", "pending test"], ["pending"]),
+ test(["a", "skipped test"], ["skipped"]),
+ ],
+ });
+
+ expect(extractFailedTests(spec, results)).toEqual([]);
+ });
+});
+
+describe("reportFailedTestsToConductor", () => {
+ const url = "https://conductor.example/webhooks";
+ const endpoint = "https://conductor.example/webhooks/failed-tests";
+ const oneTest = [{ name: "a test", file: "foo.cy.spec.ts" }];
+
+ beforeEach(() => {
+ mockFetch.mockReset();
+ mockFetch.mockResolvedValue({ ok: true, status: 200, statusText: "OK" });
+ });
+
+ it("no-ops when the webhook URL is not configured", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: undefined,
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+ expect(mockFetch).not.toHaveBeenCalled();
+ });
+
+ it("no-ops when there are no failures to report", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ });
+
+ await reportFailedTestsToConductor([]);
+ expect(mockFetch).not.toHaveBeenCalled();
+ });
+
+ it("POSTs the failures with parsed numeric ids and forwarded tests", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ REPO_ID: "123",
+ GITHUB_RUN_ID: "456",
+ GITHUB_RUN_ATTEMPT: "2",
+ CYPRESS_RETRIES: "1",
+ COMMIT_SHA: "abc123",
+ TARGET_BRANCH: "master",
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+
+ expect(mockFetch).toHaveBeenCalledTimes(1);
+ const [calledUrl, options] = mockFetch.mock.calls[0];
+ expect(calledUrl).toBe(endpoint);
+ expect(options.method).toBe("POST");
+ expect(options.headers).toMatchObject({
+ "Content-Type": "application/json",
+ });
+ expect(JSON.parse(options.body)).toEqual({
+ repo_id: 123,
+ run_id: 456,
+ attempt: 2,
+ job_id: null,
+ test_suite: "e2e",
+ sha: "abc123",
+ target_branch: "master",
+ retries: 1,
+ tests: oneTest,
+ });
+ });
+
+ it("prefers COMMIT_SHA/TARGET_BRANCH but falls back to the ambient GITHUB_* vars", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ COMMIT_SHA: undefined,
+ GITHUB_SHA: "ambient-sha",
+ TARGET_BRANCH: undefined,
+ GITHUB_BASE_REF: "ambient-branch",
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+
+ const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+ expect(body.sha).toBe("ambient-sha");
+ expect(body.target_branch).toBe("ambient-branch");
+ });
+
+ it("sends null attempt, sha, target_branch, and retries when their envs are missing", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ GITHUB_RUN_ATTEMPT: undefined,
+ CYPRESS_RETRIES: undefined,
+ COMMIT_SHA: undefined,
+ GITHUB_SHA: undefined,
+ TARGET_BRANCH: undefined,
+ GITHUB_BASE_REF: undefined,
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+
+ const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+ expect(body.attempt).toBeNull();
+ expect(body.retries).toBeNull();
+ expect(body.sha).toBeNull();
+ expect(body.target_branch).toBeNull();
+ });
+
+ it("sends the x-internal-secret header when the secret is configured", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ CI_CONDUCTOR_WEBHOOK_SECRET: "s3cr3t",
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+
+ expect(mockFetch.mock.calls[0][1].headers).toMatchObject({
+ "x-internal-secret": "s3cr3t",
+ });
+ });
+
+ it("omits the secret header when no secret is configured", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ CI_CONDUCTOR_WEBHOOK_SECRET: undefined,
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+
+ expect(mockFetch.mock.calls[0][1].headers).not.toHaveProperty(
+ "x-internal-secret",
+ );
+ });
+
+ it("appends the endpoint regardless of a trailing slash on the base URL", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: `${url}/`,
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+ expect(mockFetch.mock.calls[0][0]).toBe(endpoint);
+ });
+
+ it("uses JOB_ID env when set (populated by the resolve-job-id action)", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ JOB_ID: "789",
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+ expect(JSON.parse(mockFetch.mock.calls[0][1].body).job_id).toBe(789);
+ });
+
+ it("sends a null job_id when JOB_ID is missing or non-numeric", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ JOB_ID: "not-a-number",
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+ expect(JSON.parse(mockFetch.mock.calls[0][1].body).job_id).toBeNull();
+ });
+
+ it("sends null ids when the env vars are missing or non-numeric", async () => {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ REPO_ID: undefined,
+ GITHUB_RUN_ID: "not-a-number",
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+
+ const body = JSON.parse(mockFetch.mock.calls[0][1].body);
+ expect(body.repo_id).toBeNull();
+ expect(body.run_id).toBeNull();
+ });
+
+ it("never throws when the request fails", async () => {
+ jest.spyOn(console, "error").mockImplementation(() => {});
+ mockFetch.mockRejectedValue(new Error("network down"));
+
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ });
+
+ await expect(
+ reportFailedTestsToConductor(oneTest),
+ ).resolves.toBeUndefined();
+ expect(console.error).toHaveBeenCalled();
+ (console.error as jest.Mock).mockRestore();
+ });
+
+ it("logs the payload and does not POST in dry-run mode", async () => {
+ const logSpy = jest.spyOn(console, "log").mockImplementation(() => {});
+
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_DRY_RUN: "true",
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ REPO_ID: "123",
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+
+ expect(mockFetch).not.toHaveBeenCalled();
+ expect(logSpy).toHaveBeenCalledWith(
+ expect.stringContaining("dry run"),
+ expect.stringContaining('"repo_id":123'),
+ );
+ logSpy.mockRestore();
+ });
+
+ it("logs in dry-run mode even when no webhook URL is configured", async () => {
+ const logSpy = jest.spyOn(console, "log").mockImplementation(() => {});
+
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_DRY_RUN: "true",
+ CI_CONDUCTOR_WEBHOOK_URL: undefined,
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+
+ expect(mockFetch).not.toHaveBeenCalled();
+ expect(logSpy).toHaveBeenCalled();
+ logSpy.mockRestore();
+ });
+
+ it("logs but does not throw on a non-ok response", async () => {
+ jest.spyOn(console, "error").mockImplementation(() => {});
+ mockFetch.mockResolvedValue({
+ ok: false,
+ status: 500,
+ statusText: "Internal Server Error",
+ });
+
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ });
+
+ await reportFailedTestsToConductor(oneTest);
+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining("500"));
+ (console.error as jest.Mock).mockRestore();
+ });
+
+ it("never throws even when fetch throws synchronously", async () => {
+ jest.spyOn(console, "error").mockImplementation(() => {});
+ mockFetch.mockImplementation(() => {
+ throw new Error("synchronous boom");
+ });
+
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ });
+
+ await expect(
+ reportFailedTestsToConductor(oneTest),
+ ).resolves.toBeUndefined();
+ expect(console.error).toHaveBeenCalled();
+ (console.error as jest.Mock).mockRestore();
+ });
+});
diff --git a/e2e/support/config.js b/e2e/support/config.js
index 3324c95d03f..d4e0fc3745c 100644
--- a/e2e/support/config.js
+++ b/e2e/support/config.js
@@ -8,6 +8,10 @@ import installLogsPrinter from "cypress-terminal-report/src/installLogsPrinter";
import { BACKEND_HOST, BACKEND_PORT } from "../runner/constants/backend-port";
+import {
+ extractFailedTests,
+ reportFailedTestsToConductor,
+} from "./ci_conductor";
import * as ciTasks from "./ci_tasks";
import { collectFailingTests } from "./collectFailedTests";
import {
@@ -178,9 +182,30 @@ const defaultConfig = {
collectFailingTests(on, config);
}
- // this is an official workaround to keep recordings of the failed specs only
- // https://docs.cypress.io/guides/guides/screenshots-and-videos#Delete-videos-for-specs-without-failing-or-retried-tests
- on("after:spec", (spec, results) => {
+ // Surface the resolved Cypress retry ceiling so the ci-conductor reporter
+ // can include it in the payload (CYPRESS_RETRIES isn't otherwise set in CI;
+ // the value lives in mainConfig.retries.runMode). DEV-1999.
+ const resolvedRetries =
+ typeof config.retries === "number"
+ ? config.retries
+ : (config.retries?.runMode ?? 0);
+ process.env.CYPRESS_RETRIES = String(resolvedRetries);
+
+ on("after:spec", async (spec, results) => {
+ // Report failures to ci-conductor mid-run (no-ops unless configured).
+ if (isCI) {
+ // Reporting to ci-conductor must NEVER break the test run, so this is
+ // a hard backstop around everything — extraction, payload build, and
+ // the request. The reporter also handles its own errors internally.
+ try {
+ await reportFailedTestsToConductor(extractFailedTests(spec, results));
+ } catch (error) {
+ console.error("[ci-conductor] reporting failed (ignored)", error);
+ }
+ }
+
+ // this is an official workaround to keep recordings of the failed specs only
+ // https://docs.cypress.io/guides/guides/screenshots-and-videos#Delete-videos-for-specs-without-failing-or-retried-tests
if (results && results.video) {
// Do we have test failures?
if (results && results.video && results.stats.failures === 0) {