Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add affected tests into run tests (#73358)

Fraserc24724e78f12
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index 40b8d17081e..3191024801d 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -1,3 +1,8 @@
+# Catch-all filter — emits the full changed-files list as `all_changed_files`
+# so create-test-plan in run-tests.yml can reuse dorny's diff.
+all_changed:
+ - "**"
+
default: &default
- "locales/**"
- "bin/**/!(*.md)"
@@ -77,6 +82,19 @@ frontend_loki_ci: &frontend_loki_ci
- ".loki/**"
- ".storybook/**"
+frontend_unit_infra: &frontend_unit_infra
+ - *frontend_configs
+ - *frontend_ci
+ - "jest.config.js"
+ - "jest.tz.unit.conf.json"
+ - "frontend/test/**"
+ - "frontend/**/__mocks__/**"
+
+frontend_loki_infra: &frontend_loki_infra
+ - *frontend_configs
+ - *shared_ci
+ - *frontend_loki_ci
+
backend_ci: &backend_ci
- *shared_ci
- ".github/actions/test-driver/**"
diff --git a/.github/scripts/affected-modules.ts b/.github/scripts/affected-modules.ts
new file mode 100644
index 00000000000..51cd59e42d5
--- /dev/null
+++ b/.github/scripts/affected-modules.ts
@@ -0,0 +1,116 @@
+export type ModuleDef = { type: string; pattern: string };
+export type Rule = { from: string[]; allow: string[] };
+
+type ModuleNode = { type: string; regex: RegExp };
+
+export type ModuleGraph = {
+ nodes: ModuleNode[];
+ dependents: Map<string, Set<string>>;
+};
+
+/**
+ * Compiles a glob pattern to an anchored RegExp.
+ */
+export function globToRegex(glob: string): RegExp {
+ const escapeSegment = (seg: string) =>
+ seg.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, "[^/]*");
+ const body = glob.split("**").map(escapeSegment).join(".*");
+ return new RegExp(`^${body}$`);
+}
+
+export function buildModuleGraph(
+ elements: ModuleDef[],
+ rules: Rule[],
+): ModuleGraph {
+ const nodes = elements.map((el) => ({
+ type: el.type,
+ regex: globToRegex(el.pattern),
+ }));
+
+ const allTypes = elements.map((e) => e.type);
+
+ function expandPattern(pattern: string): string[] {
+ if (pattern === "*") {
+ return [...allTypes];
+ }
+ if (pattern.endsWith("/*")) {
+ const prefix = pattern.slice(0, -1);
+ return allTypes.filter((type) => type.startsWith(prefix));
+ }
+ return allTypes.includes(pattern) ? [pattern] : [];
+ }
+
+ // dependents.get(M) is the set of modules that depend on M
+ // i.e. modules that are affected when M changes
+ const dependents = new Map<string, Set<string>>(
+ allTypes.map((type) => [type, new Set()]),
+ );
+ for (const rule of rules) {
+ const fromTypes = rule.from.flatMap(expandPattern);
+ const allowTypes = rule.allow.flatMap(expandPattern);
+ for (const target of allowTypes) {
+ for (const importer of fromTypes) {
+ if (importer !== target) {
+ dependents.get(target)?.add(importer);
+ }
+ }
+ }
+ }
+
+ return { nodes, dependents };
+}
+
+/**
+ * Returns the module type that owns the given file.
+ * First match wins, so element ordering matters for nested patterns.
+ */
+export function mapFileToModule(
+ moduleGraph: ModuleGraph,
+ path: string,
+): string | null {
+ for (const el of moduleGraph.nodes) {
+ if (el.regex.test(path)) {
+ return el.type;
+ }
+ }
+ return null;
+}
+
+/**
+ * Returns the set of module types containing any of the changed files.
+ */
+export function getChangedModules(
+ moduleGraph: ModuleGraph,
+ changedFiles: string[],
+): Set<string> {
+ const direct = new Set<string>();
+ for (const file of changedFiles) {
+ const module = mapFileToModule(moduleGraph, file);
+ if (module) {
+ direct.add(module);
+ }
+ }
+ return direct;
+}
+
+/**
+ * Returns the changed modules and all modules that directly or indirectly depend on them
+ */
+export function getAffectedModules(
+ moduleGraph: ModuleGraph,
+ changedFiles: string[],
+): Set<string> {
+ const direct = getChangedModules(moduleGraph, changedFiles);
+ const affected = new Set(direct);
+ const queue = [...direct];
+ while (queue.length > 0) {
+ const module = queue.shift()!;
+ for (const dep of moduleGraph.dependents.get(module) ?? []) {
+ if (!affected.has(dep)) {
+ affected.add(dep);
+ queue.push(dep);
+ }
+ }
+ }
+ return affected;
+}
diff --git a/.github/scripts/affected-modules.unit.spec.ts b/.github/scripts/affected-modules.unit.spec.ts
new file mode 100644
index 00000000000..b448c20607f
--- /dev/null
+++ b/.github/scripts/affected-modules.unit.spec.ts
@@ -0,0 +1,101 @@
+import {
+ buildModuleGraph,
+ getAffectedModules,
+ getChangedModules,
+ mapFileToModule,
+} from "./affected-modules";
+
+const ELEMENTS = [
+ { type: "lib/utils", pattern: "src/utils/**" },
+ { type: "lib/types", pattern: "src/types/**" },
+ // Nested-pattern pair (mirrors mlv1-inside-mlv2): inner must come first
+ // so first-match-wins picks it up before the outer pattern.
+ { type: "lib/inner", pattern: "src/shared/v1/**" },
+ { type: "lib/outer", pattern: "src/shared/**" },
+ { type: "feature/foo", pattern: "src/foo/**" },
+ { type: "feature/bar", pattern: "src/bar/**" },
+ { type: "feature/super", pattern: "src/super/**" },
+ // Different folder root (mirrors feature/enterprise's enterprise/ path).
+ { type: "feature/external", pattern: "external/lib/**" },
+ // Same type, two patterns (mirrors app/misc per-entry-point declarations).
+ { type: "app/main", pattern: "src/app.js" },
+ { type: "app/main", pattern: "src/embed.tsx" },
+ { type: "shared/other", pattern: "src/*/**" }, // catch-all, must be last
+];
+
+const RULES = [
+ ...ELEMENTS.map((el) => ({ from: [el.type], allow: [el.type] })),
+ { from: ["lib/*"], allow: ["lib/*"] },
+ { from: ["feature/*"], allow: ["lib/*"] },
+ { from: ["feature/super"], allow: ["feature/*"] },
+ { from: ["app/*"], allow: ["lib/*", "feature/*", "app/*"] },
+];
+
+const graph = buildModuleGraph(ELEMENTS, RULES);
+
+describe("affected modules", () => {
+ describe("mapFileToModule", () => {
+ it.each([
+ ["src/utils/colors.ts", "lib/utils"],
+ ["src/types/api.ts", "lib/types"],
+ ["src/foo/foo.tsx", "feature/foo"],
+ ["src/bar/bar.tsx", "feature/bar"],
+ ["src/super/index.ts", "feature/super"],
+ ["src/app.js", "app/main"],
+ ["src/randomthing/file.ts", "shared/other"],
+ ["src/shared/v1/expr.ts", "lib/inner"],
+ ["src/shared/v2/expr.ts", "lib/outer"],
+ ["src/shared/index.ts", "lib/outer"],
+ ["external/lib/foo.ts", "feature/external"],
+ ["src/embed.tsx", "app/main"],
+ ])("should map %s to %s", (path, expected) => {
+ expect(mapFileToModule(graph, path)).toBe(expected);
+ });
+
+ it("should return null for files outside any pattern", () => {
+ expect(mapFileToModule(graph, "docs/foo.md")).toBe(null);
+ expect(mapFileToModule(graph, "README.md")).toBe(null);
+ expect(mapFileToModule(graph, "external/notlib/foo.ts")).toBe(null);
+ });
+ });
+
+ describe("getChangedModules", () => {
+ it("should return the unique set of modules whose files changed", () => {
+ const result = getChangedModules(graph, [
+ "src/utils/colors.ts",
+ "src/foo/x.ts",
+ "src/foo/y.ts",
+ "docs/unrelated.md",
+ ]);
+ expect([...result].sort()).toEqual(["feature/foo", "lib/utils"]);
+ });
+
+ it("should return an empty set when no file maps to a module", () => {
+ expect(getChangedModules(graph, ["docs/foo.md"]).size).toBe(0);
+ });
+ });
+
+ describe("getAffectedModules", () => {
+ it("should affect feature/super and app/main but not feature/bar when feature/foo changes", () => {
+ const result = getAffectedModules(graph, ["src/foo/x.ts"]);
+ expect([...result].sort()).toEqual([
+ "app/main",
+ "feature/foo",
+ "feature/super",
+ ]);
+ });
+
+ it("should affect many modules when a lib module changes", () => {
+ const result = getAffectedModules(graph, ["src/utils/colors.ts"]);
+ expect(result.has("lib/utils")).toBe(true);
+ expect(result.has("feature/foo")).toBe(true);
+ expect(result.has("feature/bar")).toBe(true);
+ expect(result.has("feature/super")).toBe(true);
+ expect(result.has("app/main")).toBe(true);
+ });
+
+ it("should return empty when no file maps to a module", () => {
+ expect(getAffectedModules(graph, ["docs/foo.md"]).size).toBe(0);
+ });
+ });
+});
diff --git a/.github/scripts/affected-tests.ts b/.github/scripts/affected-tests.ts
new file mode 100644
index 00000000000..a8f1e5f916c
--- /dev/null
+++ b/.github/scripts/affected-tests.ts
@@ -0,0 +1,140 @@
+// Per-suite affected-tests logic: combines module-affected closure with
+// per-suite "infra-touched" booleans (computed by dorny/paths-filter against
+// the `frontend_unit_infra` / `frontend_loki_infra` anchors in file-paths.yaml).
+// When infra files change, the whole suite runs in full.
+
+import {
+ type ModuleDef,
+ type ModuleGraph,
+ type Rule,
+ buildModuleGraph,
+ getAffectedModules,
+ getChangedModules,
+ mapFileToModule,
+} from "./affected-modules";
+
+export type TestSuiteDef = {
+ statsPrefix: string;
+ runAllTests?: boolean;
+};
+
+export type TestPlanStats = {
+ modules_changed: number;
+ modules_affected: number;
+ modules_affected_list: string[];
+ fe_unit_specs_total: number;
+ fe_unit_specs_run: number;
+ fe_unit_specs_skipped: number;
+ loki_stories_total: number;
+ loki_stories_run: number;
+ loki_stories_skipped: number;
+ e2e_specs_total: number;
+ e2e_specs_run: number;
+ e2e_specs_skipped: number;
+};
+
+export type TestPlan = {
+ stats: TestPlanStats;
+ fe_unit_specs_to_run: string[];
+ loki_stories_to_run: string[];
+ e2e_specs_to_run: string[];
+};
+
+export const TEST_SUITES = {
+ unit: {
+ statsPrefix: "fe_unit_specs",
+ },
+ loki: {
+ statsPrefix: "loki_stories",
+ },
+ e2e: {
+ statsPrefix: "e2e_specs",
+ // We don't map changed files to e2e tests yet, so for now just run them all
+ runAllTests: true,
+ },
+} satisfies Record<string, TestSuiteDef>;
+
+export function createTestPlan({
+ elements,
+ rules,
+ testSuites,
+ changedFiles,
+ testFilesBySuite,
+ infraTouchedBySuite,
+}: {
+ elements: ModuleDef[];
+ rules: Rule[];
+ testSuites: Record<string, TestSuiteDef>;
+ changedFiles: string[];
+ testFilesBySuite: Record<string, string[]>;
+ infraTouchedBySuite: Record<string, boolean>;
+}): TestPlan {
+ const moduleGraph = buildModuleGraph(elements, rules);
+
+ const changedModules = getChangedModules(moduleGraph, changedFiles);
+ const modulesAffected = getAffectedModules(moduleGraph, changedFiles);
+ const modulesAffectedList = [...modulesAffected].sort();
+
+ const specsToRun: Record<string, string[]> = {};
+ const stats: Record<string, number | string[]> = {
+ modules_changed: changedModules.size,
+ modules_affected: modulesAffected.size,
+ modules_affected_list: modulesAffectedList,
+ };
+
+ for (const [name, testSuite] of Object.entries(testSuites)) {
+ const files = testFilesBySuite[name] ?? [];
+ const tests = createTestPlanForSuite(
+ moduleGraph,
+ testSuite,
+ infraTouchedBySuite[name] ?? false,
+ changedFiles,
+ files,
+ );
+ specsToRun[name] = tests;
+ stats[`${testSuite.statsPrefix}_total`] = files.length;
+ stats[`${testSuite.statsPrefix}_run`] = tests.length;
+ stats[`${testSuite.statsPrefix}_skipped`] = files.length - tests.length;
+ }
+
+ return {
+ stats: stats as TestPlanStats,
+ fe_unit_specs_to_run: specsToRun.unit ?? [],
+ loki_stories_to_run: specsToRun.loki ?? [],
+ e2e_specs_to_run: specsToRun.e2e ?? [],
+ };
+}
+
+/**
+ * Computes the test plan for one test suite
+ *
+ * Runs the full suite when `runAllTests` is set or `infraTouched` is true,
+ * otherwise runs the tests based on affected-modules.
+ */
+export function createTestPlanForSuite(
+ moduleGraph: ModuleGraph,
+ testSuite: TestSuiteDef,
+ infraTouched: boolean,
+ changedFiles: string[],
+ allTestFiles: string[],
+): string[] {
+ if (testSuite.runAllTests || infraTouched) {
+ return allTestFiles;
+ }
+ const affected = getAffectedModules(moduleGraph, changedFiles);
+ return filterAffectedTests(moduleGraph, affected, allTestFiles);
+}
+
+/**
+ * Filters the test-file list down to tests whose owning module is affected.
+ */
+export function filterAffectedTests(
+ moduleGraph: ModuleGraph,
+ affected: Set<string>,
+ testFiles: string[],
+): string[] {
+ return testFiles.filter((file) => {
+ const module = mapFileToModule(moduleGraph, file);
+ return module !== null && affected.has(module);
+ });
+}
diff --git a/.github/scripts/affected-tests.unit.spec.ts b/.github/scripts/affected-tests.unit.spec.ts
new file mode 100644
index 00000000000..196fb5775a8
--- /dev/null
+++ b/.github/scripts/affected-tests.unit.spec.ts
@@ -0,0 +1,190 @@
+import { buildModuleGraph } from "./affected-modules";
+import {
+ createTestPlan,
+ createTestPlanForSuite,
+ filterAffectedTests,
+} from "./affected-tests";
+
+const ELEMENTS = [
+ { type: "lib/utils", pattern: "src/utils/**" },
+ { type: "feature/foo", pattern: "src/foo/**" },
+ { type: "feature/bar", pattern: "src/bar/**" },
+ { type: "shared/other", pattern: "src/*/**" },
+];
+
+const RULES = [
+ ...ELEMENTS.map((el) => ({ from: [el.type], allow: [el.type] })),
+ { from: ["lib/*"], allow: ["lib/*"] },
+ { from: ["feature/*"], allow: ["lib/*"] },
+];
+
+const TEST_SUITE_DEFS = {
+ unit: { statsPrefix: "fe_unit_specs" },
+ loki: { statsPrefix: "loki_stories" },
+ e2e: { statsPrefix: "e2e_specs", runAllTests: true },
+};
+
+const UNIT_FILES = [
+ "src/foo/foo.unit.spec.ts",
+ "src/foo/bar.unit.spec.ts",
+ "src/bar/bar.unit.spec.ts",
+ "src/utils/utils.unit.spec.ts",
+];
+const LOKI_FILES = ["src/foo/Foo.stories.tsx", "src/bar/Bar.stories.tsx"];
+const E2E_FILES = ["e2e/test/scenarios/a.cy.spec.ts"];
+
+const moduleGraph = buildModuleGraph(ELEMENTS, RULES);
+
+describe("affected tests", () => {
+ describe("createTestPlanForSuite", () => {
+ it("should run all tests when infra is touched", () => {
+ const result = createTestPlanForSuite(
+ moduleGraph,
+ TEST_SUITE_DEFS.unit,
+ true, // infraTouched
+ ["src/foo/x.ts"],
+ UNIT_FILES,
+ );
+ expect(result).toEqual(UNIT_FILES);
+ });
+
+ it("should fall back to affected-tests filtering when infra not touched", () => {
+ const result = createTestPlanForSuite(
+ moduleGraph,
+ TEST_SUITE_DEFS.unit,
+ false, // infraTouched
+ ["src/foo/x.ts"],
+ UNIT_FILES,
+ );
+ expect(result.sort()).toEqual([
+ "src/foo/bar.unit.spec.ts",
+ "src/foo/foo.unit.spec.ts",
+ ]);
+ });
+
+ it("should return nothing when infra not touched and no file maps to a module", () => {
+ const result = createTestPlanForSuite(
+ moduleGraph,
+ TEST_SUITE_DEFS.unit,
+ false,
+ ["docs/foo.md"],
+ UNIT_FILES,
+ );
+ expect(result).toEqual([]);
+ });
+
+ it("should run all tests for runAllTests suites regardless of inputs", () => {
+ expect(
+ createTestPlanForSuite(
+ moduleGraph,
+ TEST_SUITE_DEFS.e2e,
+ false,
+ ["docs/foo.md"],
+ E2E_FILES,
+ ),
+ ).toEqual(E2E_FILES);
+ expect(
+ createTestPlanForSuite(
+ moduleGraph,
+ TEST_SUITE_DEFS.e2e,
+ false,
+ ["src/foo/x.ts"],
+ E2E_FILES,
+ ),
+ ).toEqual(E2E_FILES);
+ });
+ });
+
+ describe("createTestPlan", () => {
+ it("should return run lists and stats together", () => {
+ const result = createTestPlan({
+ elements: ELEMENTS,
+ rules: RULES,
+ testSuites: TEST_SUITE_DEFS,
+ changedFiles: ["src/foo/x.ts"],
+ testFilesBySuite: {
+ unit: UNIT_FILES,
+ loki: LOKI_FILES,
+ e2e: E2E_FILES,
+ },
+ infraTouchedBySuite: { unit: false, loki: false, e2e: false },
+ });
+
+ expect(result.fe_unit_specs_to_run.sort()).toEqual([
+ "src/foo/bar.unit.spec.ts",
+ "src/foo/foo.unit.spec.ts",
+ ]);
+ expect(result.loki_stories_to_run).toEqual(["src/foo/Foo.stories.tsx"]);
+ expect(result.e2e_specs_to_run).toEqual(E2E_FILES);
+
+ expect(result.stats).toEqual({
+ modules_changed: 1,
+ modules_affected: 1,
+ modules_affected_list: ["feature/foo"],
+ fe_unit_specs_total: 4,
+ fe_unit_specs_run: 2,
+ fe_unit_specs_skipped: 2,
+ loki_stories_total: 2,
+ loki_stories_run: 1,
+ loki_stories_skipped: 1,
+ e2e_specs_total: 1,
+ e2e_specs_run: 1,
+ e2e_specs_skipped: 0,
+ });
+ });
+
+ it("should run all unit specs when unit infra is touched", () => {
+ const result = createTestPlan({
+ elements: ELEMENTS,
+ rules: RULES,
+ testSuites: TEST_SUITE_DEFS,
+ changedFiles: ["docs/foo.md"],
+ testFilesBySuite: {
+ unit: UNIT_FILES,
+ loki: LOKI_FILES,
+ e2e: E2E_FILES,
+ },
+ infraTouchedBySuite: { unit: true, loki: false, e2e: false },
+ });
+ expect(result.stats.fe_unit_specs_run).toBe(UNIT_FILES.length);
+ expect(result.stats.fe_unit_specs_skipped).toBe(0);
+ // Only the unit suite was triggered.
+ expect(result.stats.loki_stories_run).toBe(0);
+ });
+
+ it("should not run anything when no infra is touched and no modules affected", () => {
+ const result = createTestPlan({
+ elements: ELEMENTS,
+ rules: RULES,
+ testSuites: TEST_SUITE_DEFS,
+ changedFiles: ["docs/foo.md"],
+ testFilesBySuite: {
+ unit: UNIT_FILES,
+ loki: LOKI_FILES,
+ e2e: E2E_FILES,
+ },
+ infraTouchedBySuite: { unit: false, loki: false, e2e: false },
+ });
+ expect(result.stats.fe_unit_specs_run).toBe(0);
+ expect(result.stats.loki_stories_run).toBe(0);
+ // e2e is runAllTests, so it still reports run = total.
+ expect(result.stats.e2e_specs_run).toBe(E2E_FILES.length);
+ });
+ });
+
+ describe("filterAffectedTests", () => {
+ it("should keep only tests inside affected modules", () => {
+ const affected = new Set(["feature/foo", "lib/utils"]);
+ const tests = [
+ "src/foo/foo.unit.spec.ts",
+ "src/utils/colors.unit.spec.ts",
+ "src/bar/bar.unit.spec.ts",
+ "docs/unrelated.unit.spec.ts",
+ ];
+ expect(filterAffectedTests(moduleGraph, affected, tests)).toEqual([
+ "src/foo/foo.unit.spec.ts",
+ "src/utils/colors.unit.spec.ts",
+ ]);
+ });
+ });
+});
diff --git a/.github/scripts/create-test-plan.ts b/.github/scripts/create-test-plan.ts
new file mode 100644
index 00000000000..d36395b2935
--- /dev/null
+++ b/.github/scripts/create-test-plan.ts
@@ -0,0 +1,67 @@
+// Changed files come from dorny/paths-filter's `all_changed_files` output
+
+import { execFileSync } from "node:child_process";
+
+import { elements, rules } from "../../frontend/lint/module-boundaries.mjs";
+
+import { TEST_SUITES, createTestPlan } from "./affected-tests";
+
+const UNIT_GLOBS = [
+ "frontend/src/**/*.unit.spec.js",
+ "frontend/src/**/*.unit.spec.jsx",
+ "frontend/src/**/*.unit.spec.ts",
+ "frontend/src/**/*.unit.spec.tsx",
+ "enterprise/frontend/src/**/*.unit.spec.js",
+ "enterprise/frontend/src/**/*.unit.spec.jsx",
+ "enterprise/frontend/src/**/*.unit.spec.ts",
+ "enterprise/frontend/src/**/*.unit.spec.tsx",
+];
+
+const STORY_GLOBS = [
+ "frontend/**/*.stories.js",
+ "frontend/**/*.stories.jsx",
+ "frontend/**/*.stories.ts",
+ "frontend/**/*.stories.tsx",
+ "enterprise/frontend/**/*.stories.js",
+ "enterprise/frontend/**/*.stories.jsx",
+ "enterprise/frontend/**/*.stories.ts",
+ "enterprise/frontend/**/*.stories.tsx",
+];
+
+const E2E_GLOBS = [
+ "e2e/test/scenarios/**/*.cy.spec.js",
+ "e2e/test/scenarios/**/*.cy.spec.jsx",
+ "e2e/test/scenarios/**/*.cy.spec.ts",
+ "e2e/test/scenarios/**/*.cy.spec.tsx",
+];
+
+function listFiles(globs: string[]): string[] {
+ return execFileSync("git", ["ls-files", "--", ...globs], { encoding: "utf8" })
+ .split("\n")
+ .map((line) => line.trim())
+ .filter(Boolean);
+}
+
+const changedFiles = (process.env.CHANGED_FILES ?? "")
+ .split(",")
+ .map((s) => s.trim())
+ .filter(Boolean);
+
+const testPlan = createTestPlan({
+ elements,
+ rules,
+ testSuites: TEST_SUITES,
+ changedFiles,
+ testFilesBySuite: {
+ unit: listFiles(UNIT_GLOBS),
+ loki: listFiles(STORY_GLOBS),
+ e2e: listFiles(E2E_GLOBS),
+ },
+ infraTouchedBySuite: {
+ unit: process.env.UNIT_INFRA_TOUCHED === "true",
+ loki: process.env.LOKI_INFRA_TOUCHED === "true",
+ e2e: false,
+ },
+});
+
+process.stdout.write(JSON.stringify(testPlan) + "\n");
diff --git a/.github/scripts/upload-affected-tests-stats.js b/.github/scripts/upload-affected-tests-stats.js
new file mode 100644
index 00000000000..c1a2eb940d1
--- /dev/null
+++ b/.github/scripts/upload-affected-tests-stats.js
@@ -0,0 +1,41 @@
+// This script is used in .github/workflows/run-tests.yml.
+// It uploads the test-plan stats row produced by create-test-plan.js to
+// the stats.metabase.com tracking table.
+// Reads STATS_JSON, PR_NUMBER, HEAD_SHA, BASE_SHA, API_KEY from env.
+
+const { uploadCsvToMb } = require("./csv-to-mb.js");
+
+const stats = JSON.parse(process.env.STATS_JSON);
+console.log("stats", stats);
+
+const row = {
+ Date: new Date().toISOString(),
+ PR: Number(process.env.PR_NUMBER),
+ "Head SHA": process.env.HEAD_SHA,
+ "Base SHA": process.env.BASE_SHA,
+ "Modules changed": stats.modules_changed,
+ "Modules affected": stats.modules_affected,
+ "FE Unit specs total": stats.fe_unit_specs_total,
+ "FE Unit specs run": stats.fe_unit_specs_run,
+ "FE Unit specs skipped": stats.fe_unit_specs_skipped,
+ "Loki stories total": stats.loki_stories_total,
+ "Loki stories run": stats.loki_stories_run,
+ "Loki stories skipped": stats.loki_stories_skipped,
+ "E2E specs total": stats.e2e_specs_total,
+ "E2E specs run": stats.e2e_specs_run,
+ "E2E specs skipped": stats.e2e_specs_skipped,
+};
+
+uploadCsvToMb({
+ baseUrl: "https://stats.metabase.com",
+ tableId: 77008, // "Affected Tests Stats" table on stats.metabase.com
+ jsonData: [row],
+ mode: "append",
+})
+ .then(() => {
+ console.log("Data uploaded successfully");
+ })
+ .catch((error) => {
+ console.error("Error uploading data:", error);
+ process.exit(1);
+ });
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 59c9e5b5dfe..31f5f79bc92 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -19,9 +19,9 @@ permissions:
security-events: write
jobs:
- files-changed:
+ create-test-plan:
if: ${{ !contains(github.event.pull_request.labels.*.name,'ci skip') && !contains(github.event.pull_request.labels.*.name,'minimal-tests') }}
- name: Check which files changed
+ name: Create test plan
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 5
outputs:
@@ -38,8 +38,13 @@ jobs:
embedding_sdk_host_sample_apps: ${{ steps.changes.outputs.embedding_sdk_host_sample_apps }}
embedding_sdk_ci: ${{ steps.changes.outputs.embedding_sdk_ci }}
custom_viz_all: ${{ steps.changes.outputs.custom_viz_all }}
+ stats: ${{ steps.compute.outputs.stats }}
+ fe_unit_specs_to_run: ${{ steps.compute.outputs.fe_unit_specs_to_run }}
+ loki_stories_to_run: ${{ steps.compute.outputs.loki_stories_to_run }}
+ e2e_specs_to_run: ${{ steps.compute.outputs.e2e_specs_to_run }}
steps:
- uses: actions/checkout@v4
+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
- name: Test which files changed
uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0
id: changes
@@ -48,53 +53,87 @@ jobs:
# comma separated list of files
list-files: csv
filters: .github/file-paths.yaml
+ - name: Create test plan
+ id: compute
+ if: ${{ github.event_name == 'pull_request' }}
+ env:
+ CHANGED_FILES: ${{ steps.changes.outputs.all_changed_files }}
+ UNIT_INFRA_TOUCHED: ${{ steps.changes.outputs.frontend_unit_infra }}
+ LOKI_INFRA_TOUCHED: ${{ steps.changes.outputs.frontend_loki_infra }}
+ run: |
+ OUTPUT=$(bun .github/scripts/create-test-plan.ts)
+ echo "$OUTPUT" | jq '.stats'
+ echo "stats=$(echo "$OUTPUT" | jq -c '.stats')" >> $GITHUB_OUTPUT
+ echo "fe_unit_specs_to_run=$(echo "$OUTPUT" | jq -c '.fe_unit_specs_to_run')" >> $GITHUB_OUTPUT
+ echo "loki_stories_to_run=$(echo "$OUTPUT" | jq -c '.loki_stories_to_run')" >> $GITHUB_OUTPUT
+ echo "e2e_specs_to_run=$(echo "$OUTPUT" | jq -c '.e2e_specs_to_run')" >> $GITHUB_OUTPUT
+
+ upload-affected-tests-stats:
+ needs: create-test-plan
+ # Only land rows for the upstream repo so forks don't pollute the table.
+ if: ${{ !cancelled() && needs.create-test-plan.result == 'success' && github.event.pull_request.head.repo.full_name == 'metabase/metabase' }}
+ # Stats upload is best-effort — a failure here shouldn't fail the PR run.
+ continue-on-error: true
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 3
+ steps:
+ - uses: actions/checkout@v4
+ - name: Upload affected tests stats
+ env:
+ STATS_JSON: ${{ needs.create-test-plan.outputs.stats }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ API_KEY: ${{ secrets.STATS_API_KEY }}
+ run: node .github/scripts/upload-affected-tests-stats.js
uberjar:
- needs: [files-changed]
- if: ${{ !cancelled() && !(needs.files-changed.outputs.documentation == 'true' && needs.files-changed.outputs.backend_all == 'false' && needs.files-changed.outputs.frontend_all == 'false' && needs.files-changed.outputs.e2e_all == 'false') }}
+ needs: [create-test-plan]
+ if: ${{ !cancelled() && !(needs.create-test-plan.outputs.documentation == 'true' && needs.create-test-plan.outputs.backend_all == 'false' && needs.create-test-plan.outputs.frontend_all == 'false' && needs.create-test-plan.outputs.e2e_all == 'false') }}
uses: ./.github/workflows/uberjar.yml
secrets: inherit
backend-tests:
- if: ${{ !cancelled() && needs.files-changed.result == 'success' }}
- needs: [files-changed]
+ if: ${{ !cancelled() && needs.create-test-plan.result == 'success' }}
+ needs: [create-test-plan]
uses: ./.github/workflows/backend.yml
secrets: inherit
with:
- skip: ${{ needs.files-changed.outputs.backend_all != 'true' }}
+ skip: ${{ needs.create-test-plan.outputs.backend_all != 'true' }}
semantic-search-tests:
- needs: files-changed
+ needs: create-test-plan
uses: ./.github/workflows/semantic-search.yml
secrets: inherit
with:
- skip: ${{ needs.files-changed.outputs.backend_all != 'true' }}
+ skip: ${{ needs.create-test-plan.outputs.backend_all != 'true' }}
app-db-tests:
- needs: files-changed
+ needs: create-test-plan
uses: ./.github/workflows/app-db.yml
secrets: inherit
with:
- skip: ${{ needs.files-changed.outputs.backend_all != 'true' }}
+ skip: ${{ needs.create-test-plan.outputs.backend_all != 'true' }}
driver-tests:
- needs: files-changed
+ needs: create-test-plan
uses: ./.github/workflows/drivers.yml
secrets: inherit
with:
- skip: ${{ needs.files-changed.outputs.backend_all != 'true' }}
+ skip: ${{ needs.create-test-plan.outputs.backend_all != 'true' }}
frontend-tests:
- needs: files-changed
+ needs: [create-test-plan]
+ if: ${{ !cancelled() && needs.create-test-plan.result == 'success' }}
uses: ./.github/workflows/frontend.yml
secrets: inherit
with:
- skip: ${{ needs.files-changed.outputs.frontend_all != 'true' }}
- skip-lint: ${{ needs.files-changed.outputs.frontend_all != 'true' && needs.files-changed.outputs.e2e_all != 'true' }}
- skip-custom-viz: ${{ needs.files-changed.outputs.custom_viz_all != 'true' }}
+ skip: ${{ needs.create-test-plan.outputs.frontend_all != 'true' }}
+ skip-lint: ${{ needs.create-test-plan.outputs.frontend_all != 'true' && needs.create-test-plan.outputs.e2e_all != 'true' }}
+ skip-custom-viz: ${{ needs.create-test-plan.outputs.custom_viz_all != 'true' }}
e2e-tests:
- needs: [files-changed, uberjar]
+ needs: [create-test-plan, uberjar]
# Skip e2e tests when the uberjar build failed or was cancelled — the
# downstream tests can't fetch a missing artifact, and we don't want to
# mask the uberjar failure with a wave of misleading e2e errors. We still
@@ -105,43 +144,43 @@ jobs:
# branch protection blocks the merge as pending.
if: |
always() && !cancelled() &&
- needs.files-changed.result == 'success' &&
+ needs.create-test-plan.result == 'success' &&
needs.uberjar.result != 'failure' &&
needs.uberjar.result != 'cancelled'
uses: ./.github/workflows/e2e-tests.yml
secrets: inherit
with:
- skip: ${{ needs.files-changed.outputs.e2e_all != 'true' }}
- # here we pass result from files-changed, which is a list of files, comma separated
+ skip: ${{ needs.create-test-plan.outputs.e2e_all != 'true' }}
+ # here we pass the list of changed files
# e.g. e2e/test/scenarios/onboarding/command-palette.cy.spec.js,e2e/test/scenarios/question/document-title.cy.spec.js
# e2e_all_files includes all changes which trigger all e2e tests, if the list of changes files is included in the
# list of changed specs files, then only specs are changed and we can run only those specs
- specs: ${{ needs.files-changed.outputs.e2e_all_files == needs.files-changed.outputs.e2e_specs_files && needs.files-changed.outputs.e2e_specs_files || '' }}
+ specs: ${{ needs.create-test-plan.outputs.e2e_all_files == needs.create-test-plan.outputs.e2e_specs_files && needs.create-test-plan.outputs.e2e_specs_files || '' }}
embedding-sdk-package:
- needs: [files-changed]
+ needs: [create-test-plan]
if: |
!cancelled() &&
- (needs.files-changed.outputs.embedding_sdk_components == 'true' ||
- needs.files-changed.outputs.frontend_sources == 'true' ||
- needs.files-changed.outputs.embedding_documentation == 'true' ||
- needs.files-changed.outputs.embedding_sdk_ci == 'true' ||
- needs.files-changed.outputs.embedding_sdk_host_sample_apps == 'true')
+ (needs.create-test-plan.outputs.embedding_sdk_components == 'true' ||
+ needs.create-test-plan.outputs.frontend_sources == 'true' ||
+ needs.create-test-plan.outputs.embedding_documentation == 'true' ||
+ needs.create-test-plan.outputs.embedding_sdk_ci == 'true' ||
+ needs.create-test-plan.outputs.embedding_sdk_host_sample_apps == 'true')
uses: ./.github/workflows/embedding-sdk-package-build.yml
secrets: inherit
sdk-tests:
- needs: [uberjar, embedding-sdk-package, files-changed]
+ needs: [uberjar, embedding-sdk-package, create-test-plan]
if: always() && !cancelled()
uses: ./.github/workflows/embedding-sdk.yml
secrets: inherit
with:
- bundle-size-check: ${{ github.event_name == 'pull_request' && needs.uberjar.result == 'success' && needs.files-changed.outputs.frontend_sources == 'true' }}
- cli-snippets-type-check: ${{ needs.files-changed.outputs.frontend_sources == 'true' }}
- docs-snippets-type-check: ${{ needs.files-changed.outputs.embedding_documentation == 'true' }}
- documentation-validation: ${{ needs.files-changed.outputs.frontend_sources == 'true' }}
- component-tests: ${{ needs.files-changed.outputs.embedding_sdk_components == 'true' }}
- host-sample-apps-tests: ${{ needs.files-changed.outputs.embedding_sdk_host_sample_apps == 'true' }}
+ bundle-size-check: ${{ github.event_name == 'pull_request' && needs.uberjar.result == 'success' && needs.create-test-plan.outputs.frontend_sources == 'true' }}
+ cli-snippets-type-check: ${{ needs.create-test-plan.outputs.frontend_sources == 'true' }}
+ docs-snippets-type-check: ${{ needs.create-test-plan.outputs.embedding_documentation == 'true' }}
+ documentation-validation: ${{ needs.create-test-plan.outputs.frontend_sources == 'true' }}
+ component-tests: ${{ needs.create-test-plan.outputs.embedding_sdk_components == 'true' }}
+ host-sample-apps-tests: ${{ needs.create-test-plan.outputs.embedding_sdk_host_sample_apps == 'true' }}
skip: ${{ needs.embedding-sdk-package.result != 'success' || needs.uberjar.result != 'success' }}
pr-env:
diff --git a/tsconfig.json b/tsconfig.json
index 2e2352f8489..1161c58aec1 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -30,7 +30,8 @@
"frontend/build/**/*",
"enterprise/frontend/src/**/*",
"frontend/test/**/*",
- "enterprise/frontend/test/**/*"
+ "enterprise/frontend/test/**/*",
+ ".github/scripts/**/*"
],
"exclude": [
"node_modules",