Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Send Cypress failure screenshots to ci-conductor (#75283)

Nemanja Glumac728296f25ba0
diff --git a/e2e/support/ci_conductor.ts b/e2e/support/ci_conductor.ts
index 1f6797d4c0b..7aea902984a 100644
--- a/e2e/support/ci_conductor.ts
+++ b/e2e/support/ci_conductor.ts
@@ -1,3 +1,5 @@
+import { readFileSync, statSync } from "node:fs";
+
import fetch from "node-fetch"; // must be node-fetch v2 because it's non-esm
// `Cypress` and `CypressCommandLine` are global namespaces provided by the
@@ -57,6 +59,19 @@ type ConductorTest = {
* value. Conductor schema isn't final — fields it doesn't store are ignored.
*/
message?: string | null;
+ /**
+ * The test's first failure screenshot as a base64 PNG data URI. ci-conductor
+ * uploads it to S3 and stores the public URL. Attached at send time by
+ * `reportFailedTestsToConductor`; absent when there's no screenshot or it
+ * can't be read. See DEV-2000.
+ */
+ failure_screenshot?: string;
+ /**
+ * Transient (never sent on the wire): absolute path to the resolved first
+ * failure screenshot, set by `extractFailedTests`. It's read and encoded into
+ * `failure_screenshot` just before the POST, then dropped. See DEV-2000.
+ */
+ screenshotPath?: string;
};
/**
@@ -95,6 +110,89 @@ function toNumber(value: string | undefined): number | null {
return Number.isFinite(parsed) ? parsed : null;
}
+/** Normalize a string to just [a-z0-9] for sanitization-proof matching. */
+function normalizeForMatch(value: string): string {
+ return value.toLowerCase().replace(/[^a-z0-9]/g, "");
+}
+
+/** Basename of a path, tolerant of both POSIX and Windows separators. */
+function baseName(p: string): string {
+ return p.split(/[\\/]/).pop() ?? "";
+}
+
+/**
+ * Match a failed test to its first-failure screenshot from after:spec's
+ * `results.screenshots` — a flat, spec-level list with no link back to the test.
+ * Cypress derives each screenshot filename from the test's full title (joined
+ * with " -- ", sanitized) followed by " (failed)" (plus "(attempt N)" for
+ * retries). We normalize the title and each basename down to [a-z0-9] — which
+ * cancels every sanitization rule (">", " -- ", "/", ":", ...) on both sides, so
+ * we never reconstruct the filename — and anchor on the title immediately
+ * followed by "failed":
+ *
+ * normalize(basename) === normalize(title.join("")) + "failed" + [attempt…] + ext
+ *
+ * Anchoring on "failed" (a) excludes manual cy.screenshot() shots, which have no
+ * "(failed)" suffix, and (b) is collision-proof: a title that's a prefix of
+ * another's won't match, because "failed" must follow the key exactly. The first
+ * shot and its retries both anchor-match, so we take the shortest basename — the
+ * one without an "(attempt N)" suffix, i.e. the first attempt.
+ *
+ * Pure and defensive: returns undefined on any missing/odd input, never throws.
+ * See DEV-2000.
+ */
+export function resolveScreenshotPath(
+ titlePath: string[] | undefined,
+ screenshots: Array<{ path?: string }> | undefined,
+): string | undefined {
+ try {
+ const key = normalizeForMatch((titlePath ?? []).join(""));
+ if (!key || !Array.isArray(screenshots)) {
+ return undefined;
+ }
+
+ const anchor = `${key}failed`;
+ return (
+ screenshots
+ .map((shot) => (typeof shot?.path === "string" ? shot.path : ""))
+ .filter((path) => path !== "")
+ .filter((path) => normalizeForMatch(baseName(path)).startsWith(anchor))
+ // Shortest basename = the first attempt (no "(attempt N)" suffix).
+ .sort((a, b) => baseName(a).length - baseName(b).length)[0]
+ );
+ } catch (error) {
+ console.error("[ci-conductor] failed to resolve screenshot path", error);
+ return undefined;
+ }
+}
+
+// ci-conductor rejects screenshots over 10 MB decoded; base64 inflates ~33%, so
+// we cap the raw file well under that (see its lib/screenshots MAX_SCREENSHOT_BYTES).
+const MAX_SCREENSHOT_RAW_BYTES = 7 * 1024 * 1024;
+
+/**
+ * Read a screenshot file and return it as a base64 PNG data URI, or undefined if
+ * the file is missing, empty, too large, or unreadable. Best-effort and never
+ * throws — a screenshot problem must not break reporting. See DEV-2000.
+ */
+function encodeScreenshot(filePath: string): string | undefined {
+ try {
+ const stat = statSync(filePath);
+ if (
+ !stat.isFile() ||
+ stat.size === 0 ||
+ stat.size > MAX_SCREENSHOT_RAW_BYTES
+ ) {
+ return undefined;
+ }
+ const b64 = readFileSync(filePath).toString("base64");
+ return b64 ? `data:image/png;base64,${b64}` : undefined;
+ } catch (error) {
+ console.error("[ci-conductor] failed to read screenshot", filePath, error);
+ return undefined;
+ }
+}
+
/**
* Pull the reportable tests out of a single spec's run results. We always
* include any test that had at least one failed attempt — so both "broken"
@@ -139,6 +237,14 @@ export function extractFailedTests(
const name = titlePath[titlePath.length - 1] ?? "(unknown test)";
const suite = titlePath.slice(0, -1).join(" > ");
const attempts = test.attempts ?? [];
+ // Resolve (but don't yet read) the test's first failure screenshot from
+ // this spec's `results.screenshots`. The file is encoded into
+ // `failure_screenshot` at send time so the matching logic here stays pure.
+ // Returns undefined on any miss — best-effort.
+ const screenshotPath = resolveScreenshotPath(
+ titlePath,
+ results?.screenshots,
+ );
return {
name,
path: suite || undefined,
@@ -147,6 +253,7 @@ export function extractFailedTests(
attempts,
status: classifyStatus(attempts),
message: test.displayError ?? null,
+ ...(screenshotPath ? { screenshotPath } : {}),
};
});
@@ -181,6 +288,17 @@ export async function reportFailedTestsToConductor(
// Everything below is wrapped so the reporter can never throw into the test
// run, regardless of payload contents or network behavior.
try {
+ // Best-effort: read each resolved screenshot and inline it as base64 for
+ // ci-conductor to upload. encodeScreenshot never throws — a test simply goes
+ // without a screenshot if it can't be read. The transient `screenshotPath`
+ // is dropped here so it never reaches the wire.
+ const outgoing = tests.map(({ screenshotPath, ...rest }) => {
+ const failure_screenshot = screenshotPath
+ ? encodeScreenshot(screenshotPath)
+ : undefined;
+ return failure_screenshot ? { ...rest, failure_screenshot } : rest;
+ });
+
const body = {
repo_id: toNumber(REPO_ID),
run_id: toNumber(GITHUB_RUN_ID),
@@ -195,13 +313,25 @@ export async function reportFailedTestsToConductor(
// CYPRESS_RETRIES isn't set in CI by default; e2e/support/config.js
// surfaces the resolved Cypress config value into this env at startup.
retries: toNumber(process.env.CYPRESS_RETRIES),
- tests,
+ tests: outgoing,
};
if (isDryRun) {
+ // Don't dump base64 blobs into the log — replace each with a size marker.
+ const screenshotCount = outgoing.filter(
+ (t) => t.failure_screenshot,
+ ).length;
+ const redactedTests = outgoing.map((t) =>
+ t.failure_screenshot
+ ? {
+ ...t,
+ failure_screenshot: `<base64 ${t.failure_screenshot.length} chars>`,
+ }
+ : t,
+ );
console.log(
- `[ci-conductor] (dry run) would POST ${tests.length} failure(s):`,
- JSON.stringify(body),
+ `[ci-conductor] (dry run) would POST ${outgoing.length} failure(s), ${screenshotCount} with screenshot(s):`,
+ JSON.stringify({ ...body, tests: redactedTests }),
);
return;
}
diff --git a/e2e/support/ci_conductor.unit.spec.ts b/e2e/support/ci_conductor.unit.spec.ts
index c6dce2ddbfb..7fab957b795 100644
--- a/e2e/support/ci_conductor.unit.spec.ts
+++ b/e2e/support/ci_conductor.unit.spec.ts
@@ -1,4 +1,4 @@
-import { extractFailedTests } from "./ci_conductor";
+import { extractFailedTests, resolveScreenshotPath } from "./ci_conductor";
// jest.mock is hoisted; the mocked default must be referenced via a `mock`-
// prefixed variable. node-fetch is unused by extractFailedTests, so mocking it
@@ -69,6 +69,13 @@ const test = (
attempts: attemptStates.map((state) => ({ state })),
});
+// Build a `results.screenshots`-shaped array from bare paths (the only field we
+// read). Cypress leaves `name` null for automatic failure shots.
+const screenshotsFromPaths = (...paths: string[]) =>
+ paths.map((path) => ({
+ path,
+ })) as unknown as CypressCommandLine.RunResult["screenshots"];
+
describe("extractFailedTests", () => {
// Pin the default to "first attempt" so these tests are deterministic even
// when the CI job running them is itself a re-run (GITHUB_RUN_ATTEMPT > 1).
@@ -243,6 +250,102 @@ describe("extractFailedTests", () => {
expect(extractFailedTests(spec, results)).toEqual([]);
});
+
+ it("attaches screenshotPath for a failed test with a matching screenshot", () => {
+ const path =
+ "/cy/screenshots/foo.cy.spec.js/a -- shows an error (failed).png";
+ const results = makeResults({
+ tests: [test(["a", "shows an error"], ["failed"], "boom")],
+ screenshots: screenshotsFromPaths(path),
+ });
+
+ expect(extractFailedTests(spec, results)[0]).toMatchObject({
+ name: "shows an error",
+ screenshotPath: path,
+ });
+ });
+
+ it("omits screenshotPath when no screenshot matches the test", () => {
+ const unrelated =
+ "/cy/screenshots/foo/a -- a totally different test (failed).png";
+ const results = makeResults({
+ tests: [test(["a", "shows an error"], ["failed"], "boom")],
+ screenshots: screenshotsFromPaths(unrelated),
+ });
+
+ expect(extractFailedTests(spec, results)[0]).not.toHaveProperty(
+ "screenshotPath",
+ );
+ });
+
+ it("omits screenshotPath when the spec produced no screenshots", () => {
+ const results = makeResults({
+ tests: [test(["a", "shows an error"], ["failed"], "boom")],
+ });
+
+ expect(extractFailedTests(spec, results)[0]).not.toHaveProperty(
+ "screenshotPath",
+ );
+ });
+});
+
+describe("resolveScreenshotPath", () => {
+ const shots = (...paths: string[]) => paths.map((path) => ({ path }));
+
+ it("returns undefined with no screenshots, no title, or no match", () => {
+ expect(resolveScreenshotPath(["a", "b"], undefined)).toBeUndefined();
+ expect(resolveScreenshotPath(["a", "b"], [])).toBeUndefined();
+ expect(
+ resolveScreenshotPath(undefined, shots("/x/a (failed).png")),
+ ).toBeUndefined();
+ expect(
+ resolveScreenshotPath(["a", "b"], shots("/x/unrelated (failed).png")),
+ ).toBeUndefined();
+ });
+
+ it("matches the real on-disk basename despite spaces, ` -- ` and `>` stripping", () => {
+ // Verbatim from a local run: describe("scenarios > models > create") →
+ // Cypress writes "scenarios models create -- <test> (failed).png".
+ const path =
+ "/home/dev/code/metabase/cypress/screenshots/create.cy.spec.js/scenarios models create -- user without a collection access should still be able to create and save a model in his own personal collection (failed).png";
+ expect(
+ resolveScreenshotPath(
+ [
+ "scenarios > models > create",
+ "user without a collection access should still be able to create and save a model in his own personal collection",
+ ],
+ shots(path),
+ ),
+ ).toBe(path);
+ });
+
+ it("excludes manual screenshots (no `(failed)` anchor)", () => {
+ // A manual cy.screenshot() is named "<title>.png" — no "(failed)".
+ const manual = "/x/a -- renders the chart.png";
+ expect(
+ resolveScreenshotPath(["a", "renders the chart"], shots(manual)),
+ ).toBeUndefined();
+ });
+
+ it("picks the first attempt over its retry", () => {
+ const first = "/x/a -- flaky (failed).png";
+ const retry = "/x/a -- flaky (failed) (attempt 2).png";
+ expect(resolveScreenshotPath(["a", "flaky"], shots(retry, first))).toBe(
+ first,
+ );
+ });
+
+ it("anchors on `(failed)`, so a prefix title can't steal a longer test's shot", () => {
+ const shortShot = "/x/a -- loads (failed).png";
+ const longShot = "/x/a -- loads quickly (failed).png";
+ // "loads" anchors on "loadsfailed"; "loadsquickly…" can't match it.
+ expect(
+ resolveScreenshotPath(["a", "loads"], shots(longShot, shortShot)),
+ ).toBe(shortShot);
+ expect(
+ resolveScreenshotPath(["a", "loads quickly"], shots(longShot, shortShot)),
+ ).toBe(longShot);
+ });
});
describe("reportFailedTestsToConductor", () => {
@@ -477,6 +580,52 @@ describe("reportFailedTestsToConductor", () => {
(console.error as jest.Mock).mockRestore();
});
+ it("inlines a resolved screenshot as base64 and drops screenshotPath", async () => {
+ const fs = await import("node:fs");
+ const os = await import("node:os");
+ const pathMod = await import("node:path");
+ const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); // PNG magic
+ const file = pathMod.join(os.tmpdir(), "ci-conductor-shot.png");
+ fs.writeFileSync(file, bytes);
+ try {
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ });
+ await reportFailedTestsToConductor([
+ { name: "t", file: "f.cy.spec.ts", screenshotPath: file },
+ ]);
+ const sent = JSON.parse(mockFetch.mock.calls[0][1].body).tests[0];
+ expect(sent.failure_screenshot).toBe(
+ `data:image/png;base64,${bytes.toString("base64")}`,
+ );
+ expect(sent).not.toHaveProperty("screenshotPath");
+ } finally {
+ fs.unlinkSync(file);
+ }
+ });
+
+ it("omits the screenshot (and screenshotPath) when the file can't be read", async () => {
+ const errorSpy = jest.spyOn(console, "error").mockImplementation(() => {});
+ const { reportFailedTestsToConductor } = await loadConductor({
+ CI_CONDUCTOR_WEBHOOK_URL: url,
+ });
+
+ await reportFailedTestsToConductor([
+ { name: "t", file: "f.cy.spec.ts", screenshotPath: "/no/such/file.png" },
+ ]);
+
+ const sent = JSON.parse(mockFetch.mock.calls[0][1].body).tests[0];
+ expect(sent).not.toHaveProperty("failure_screenshot");
+ expect(sent).not.toHaveProperty("screenshotPath");
+ // The read failure is logged (best-effort), not thrown.
+ expect(errorSpy).toHaveBeenCalledWith(
+ expect.stringContaining("failed to read screenshot"),
+ "/no/such/file.png",
+ expect.anything(),
+ );
+ errorSpy.mockRestore();
+ });
+
it("never throws even when fetch throws synchronously", async () => {
jest.spyOn(console, "error").mockImplementation(() => {});
mockFetch.mockImplementation(() => {