Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use affected-test plans to narrow Loki visual tests (#82050)

Frasere677b6b267da
diff --git a/.dependency-cruiser.cjs b/.dependency-cruiser.cjs
index f2ffdfc9949..488c0a118a5 100644
--- a/.dependency-cruiser.cjs
+++ b/.dependency-cruiser.cjs
@@ -2,7 +2,7 @@
// imported, vs the *allowed* edges in module-boundaries.mjs). Consumed by
// .github/scripts/create-test-plan.ts.
//
-// bunx depcruise frontend/src enterprise/frontend/src \
+// bunx depcruise frontend/src enterprise/frontend/src .storybook/preview.tsx \
// --config .dependency-cruiser.cjs --output-type json > dependency-graph.json
/** @type {import('dependency-cruiser').IConfiguration} */
@@ -10,7 +10,7 @@ module.exports = {
options: {
// Keep only edges between our own source files (drops node_modules etc.),
// which keeps the JSON small.
- includeOnly: "^(frontend/src|enterprise/frontend/src)/",
+ includeOnly: "^(frontend/src|enterprise/frontend/src|\\.storybook)/",
doNotFollow: { path: "node_modules" },
diff --git a/.github/actions/create-test-plan/action.yml b/.github/actions/create-test-plan/action.yml
index b8009362668..19eec87739c 100644
--- a/.github/actions/create-test-plan/action.yml
+++ b/.github/actions/create-test-plan/action.yml
@@ -99,6 +99,10 @@ runs:
echo
echo "Changed: $(jq '.stats.fe_files_changed' "$plan") FE / $(jq '.stats.be_files_changed' "$plan") BE file(s) · $(jq '.stats.fe_modules_affected_usage' "$plan") FE module(s) affected"
echo
+ if [ "$(jq '.stats.loki_preview_touched' "$plan")" = "true" ]; then
+ echo "**Loki: full run**: a file the Storybook preview imports changed."
+ echo
+ fi
e2e_run=$(jq '.stats.e2e_specs_to_run_usage' "$plan")
e2e_all=$(jq '.stats.e2e_specs_all' "$plan")
if [ "$e2e_run" -eq "$e2e_all" ]; then
diff --git a/.github/actions/prepare-test-selection/action.yml b/.github/actions/prepare-test-selection/action.yml
new file mode 100644
index 00000000000..fafa612cf70
--- /dev/null
+++ b/.github/actions/prepare-test-selection/action.yml
@@ -0,0 +1,39 @@
+name: Prepare test selection
+description: Download this run's test plan and write the suite's selected test paths.
+
+inputs:
+ suite:
+ description: The suite to select tests for (unit, loki or e2e).
+ required: true
+ test-plan-artifact-id:
+ description: ID of this run's test-plan.json artifact.
+ required: true
+ require-download:
+ description: Fail the step when the test plan download fails, instead of running the full suite.
+ required: false
+ default: "false"
+
+outputs:
+ paths-file:
+ description: JSON file listing the selected test paths, or '' when the suite runs in full.
+ value: ${{ steps.selection.outputs.paths-file }}
+ selection:
+ description: full, narrowed or empty, or '' when the plan is unavailable and the suite runs in full.
+ value: ${{ steps.selection.outputs.selection }}
+
+runs:
+ using: composite
+ steps:
+ - name: Download test plan
+ id: test-plan
+ continue-on-error: ${{ inputs.require-download != 'true' }}
+ uses: actions/download-artifact@v8
+ with:
+ artifact-ids: ${{ inputs.test-plan-artifact-id }}
+ path: ${{ runner.temp }}/test-plan
+ - name: Prepare test selection
+ id: selection
+ shell: bash
+ env:
+ PLAN_DOWNLOADED: ${{ steps.test-plan.outcome }}
+ run: bun .github/scripts/prepare-test-selection.ts "${{ inputs.suite }}"
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index 6424003df5a..db7b2d97d20 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -78,6 +78,7 @@ frontend_all: &frontend_all
frontend_loki_ci: &frontend_loki_ci
- ".github/workflows/loki.yml"
+ - "loki.config.js"
- ".loki/**"
- ".storybook/**"
@@ -96,6 +97,19 @@ frontend_loki_infra: &frontend_loki_infra
- *frontend_configs
- *shared_ci
- *frontend_loki_ci
+ - "frontend/build/shared/**"
+ - "frontend/test/__support__/**"
+ - "patches/**"
+ # dependency-cruiser doesn't follow CSS @import, so global styles must run every story.
+ - "frontend/src/metabase/css/**/*.css"
+ # Storybook serves resources/frontend_client as static files, and the stories bundle the app's SVGs.
+ - "resources/frontend_client/**"
+ - "frontend/**/*.svg"
+ - "enterprise/frontend/**/*.svg"
+
+frontend_loki_all:
+ - *frontend_sources
+ - *frontend_loki_infra
# Everything the ci-scripts jest project runs or depends on.
# test-unit ignores that project, so this filter is the only thing that reruns it.
@@ -107,6 +121,8 @@ ci_scripts:
- ".github/file-paths.yaml"
- ".github/scripts/**"
- ".github/actions/**"
+ - ".github/workflows/loki.yml"
+ - ".storybook/story-files.cjs"
- "frontend/lint/module-boundaries.mjs"
- "frontend/lint/shared-tiers.mjs"
- "frontend/test/jest-test-paths-filter.ts"
diff --git a/.github/scripts/affected-modules.ts b/.github/scripts/affected-modules.ts
index 98b4db8b9dd..f0456369fe9 100644
--- a/.github/scripts/affected-modules.ts
+++ b/.github/scripts/affected-modules.ts
@@ -105,6 +105,28 @@ export function getAffectedFiles(
return affected;
}
+/** The file plus every file it transitively imports. */
+export function getTransitiveDependencies(
+ fileDependencies: FileDependency[],
+ file: string,
+): Set<string> {
+ const dependenciesOf = new Map(
+ fileDependencies.map(({ source, dependencies }) => [source, dependencies]),
+ );
+ const reached = new Set([file]);
+ const queue = [file];
+ while (queue.length > 0) {
+ const current = queue.shift()!;
+ for (const dependency of dependenciesOf.get(current) ?? []) {
+ if (!reached.has(dependency)) {
+ reached.add(dependency);
+ queue.push(dependency);
+ }
+ }
+ }
+ return reached;
+}
+
export type ModuleGraph = {
nodes: ModuleNode[];
/**
diff --git a/.github/scripts/affected-modules.unit.spec.ts b/.github/scripts/affected-modules.unit.spec.ts
index e98aaaff0dd..09903107c77 100644
--- a/.github/scripts/affected-modules.unit.spec.ts
+++ b/.github/scripts/affected-modules.unit.spec.ts
@@ -6,6 +6,7 @@ import {
getAffectedFiles,
getAffectedModules,
getChangedModules,
+ getTransitiveDependencies,
mapFileToModule,
parseCruiseModules,
} from "./affected-modules";
@@ -123,6 +124,31 @@ describe("affected modules", () => {
});
});
+ describe("getTransitiveDependencies", () => {
+ it("should return the file and everything it transitively imports", () => {
+ expect(
+ [...getTransitiveDependencies(FILE_DEPS, "src/app.js")].sort(),
+ ).toEqual(["src/app.js", "src/foo/foo.tsx", "src/utils/colors.ts"]);
+ });
+
+ it("should return only the file when it imports nothing", () => {
+ expect([
+ ...getTransitiveDependencies(FILE_DEPS, "src/utils/colors.ts"),
+ ]).toEqual(["src/utils/colors.ts"]);
+ });
+
+ it("should stop at an import cycle", () => {
+ const cyclic = [
+ { source: "src/a.ts", dependencies: ["src/b.ts"] },
+ { source: "src/b.ts", dependencies: ["src/a.ts"] },
+ ];
+
+ expect([...getTransitiveDependencies(cyclic, "src/a.ts")].sort()).toEqual(
+ ["src/a.ts", "src/b.ts"],
+ );
+ });
+ });
+
describe("buildUsageModuleGraph + getAffectedModules", () => {
const usageGraph = buildUsageModuleGraph(ELEMENTS, FILE_DEPS);
diff --git a/.github/scripts/affected-tests.ts b/.github/scripts/affected-tests.ts
index f9f36e0d41f..ff3416cda00 100644
--- a/.github/scripts/affected-tests.ts
+++ b/.github/scripts/affected-tests.ts
@@ -11,6 +11,7 @@ import {
buildUsageModuleGraph,
getAffectedModules,
getChangedModules,
+ getTransitiveDependencies,
mapFileToModule,
} from "./affected-modules";
@@ -21,6 +22,7 @@ export type TestPlanStats = {
be_files_total: number;
unit_infra_touched: boolean;
loki_infra_touched: boolean;
+ loki_preview_touched: boolean;
shared_sources_touched: boolean;
fe_modules_total: number;
fe_modules_changed: number;
@@ -52,6 +54,8 @@ export type CreateTestPlanInput = {
// Null falls back to the rules graph.
loadFileDependencies: () => FileDependency[] | null;
testFilesBySuite: { unit: string[]; loki: string[]; e2e: string[] };
+ // Storybook's preview file, whose imports wrap every story.
+ lokiPreviewFile: string;
e2eSpecFiles: Record<string, string[]> | null;
unitInfraTouched: boolean;
lokiInfraTouched: boolean;
@@ -134,6 +138,7 @@ export function createTestPlan({
changedFiles,
loadFileDependencies,
testFilesBySuite,
+ lokiPreviewFile,
e2eSpecFiles,
unitInfraTouched,
lokiInfraTouched,
@@ -158,7 +163,7 @@ export function createTestPlan({
// cljc/cljs compile into the FE bundle, so they force a full run that module
// selection can't narrow — same effect as a suite's own infra changing.
const unitForceAll = unitInfraTouched || sharedSourcesTouched;
- const lokiForceAll = lokiInfraTouched || sharedSourcesTouched;
+ const lokiInfraForceAll = lokiInfraTouched || sharedSourcesTouched;
// e2e is integration-level, so anything the FE-coverage manifest can't see
// forces a full run: cljc/cljs in the bundle (sharedSourcesTouched), a backend
// change that can break the UI (beFilesChanged), an e2e harness/support change
@@ -171,12 +176,26 @@ export function createTestPlan({
// The cruise takes seconds, so it is skipped when every suite runs in full anyway.
const edges =
- unitForceAll && lokiForceAll && e2eForceAll ? null : loadFileDependencies();
+ unitForceAll && lokiInfraForceAll && e2eForceAll
+ ? null
+ : loadFileDependencies();
const usageGraph = edges
? buildUsageModuleGraph(elements, edges)
: rulesGraph;
const usageAffected = getAffectedModules(usageGraph, changedFiles);
+ // The preview wraps every story, so a change to a file it imports can change any screenshot.
+ const previewDependencies = edges
+ ? getTransitiveDependencies(edges, lokiPreviewFile)
+ : new Set<string>();
+ const lokiPreviewTouched = changedFiles.some((file) =>
+ previewDependencies.has(file),
+ );
+ // Without the cruise, the rules graph cannot account for the preview's
+ // global dependencies, so it cannot safely narrow Loki.
+ const lokiForceAll =
+ lokiInfraForceAll || lokiPreviewTouched || edges === null;
+
const select = (forceAll: boolean, affected: Set<string>, files: string[]) =>
forceAll ? files : filterAffectedTests(nodes, affected, files);
@@ -213,6 +232,7 @@ export function createTestPlan({
be_files_total: beFilesTotal,
unit_infra_touched: unitInfraTouched,
loki_infra_touched: lokiInfraTouched,
+ loki_preview_touched: lokiPreviewTouched,
shared_sources_touched: sharedSourcesTouched,
fe_modules_total: totalModules,
fe_modules_changed: changedModules.size,
diff --git a/.github/scripts/affected-tests.unit.spec.ts b/.github/scripts/affected-tests.unit.spec.ts
index 64a89e72bd2..21d460de80c 100644
--- a/.github/scripts/affected-tests.unit.spec.ts
+++ b/.github/scripts/affected-tests.unit.spec.ts
@@ -28,6 +28,7 @@ const baseInput = {
rules: RULES,
loadFileDependencies: () => null,
testFilesBySuite: { unit: UNIT_FILES, loki: LOKI_FILES, e2e: E2E_FILES },
+ lokiPreviewFile: ".storybook/preview.tsx",
e2eSpecFiles: null,
unitInfraTouched: false,
lokiInfraTouched: false,
@@ -41,7 +42,10 @@ const baseInput = {
describe("createTestPlan", () => {
it("runs only specs in affected modules (rules graph)", () => {
- const plan = createTestPlan({ ...baseInput, changedFiles: ["src/foo/x.ts"] });
+ const plan = createTestPlan({
+ ...baseInput,
+ changedFiles: ["src/foo/x.ts"],
+ });
expect(plan.stats.fe_modules_changed).toBe(1);
expect(plan.stats.fe_modules_affected_rules).toBe(1);
@@ -50,8 +54,8 @@ describe("createTestPlan", () => {
"src/foo/foo.unit.spec.ts",
]);
expect(plan.stats.unit_specs_to_run_rules).toBe(2);
- expect(plan.stats.loki_stories_to_run_rules).toBe(1);
- // With no file deps, usage falls back to rules.
+ expect(plan.loki_stories_to_run).toEqual(LOKI_FILES);
+ // With no file deps, unit usage falls back to rules, but Loki runs in full.
expect(plan.stats.unit_specs_to_run_usage).toBe(2);
});
@@ -91,14 +95,69 @@ describe("createTestPlan", () => {
...baseInput,
changedFiles: ["docs/readme.md"],
unitInfraTouched: true,
+ loadFileDependencies: () => [],
});
expect(plan.stats.unit_specs_to_run_rules).toBe(UNIT_FILES.length);
expect(plan.stats.loki_stories_to_run_rules).toBe(0);
});
+ // The preview imports the theme, which imports the colors.
+ const PREVIEW_DEPS = [
+ { source: ".storybook/preview.tsx", dependencies: ["src/utils/theme.ts"] },
+ { source: "src/utils/theme.ts", dependencies: ["src/utils/colors.ts"] },
+ { source: "src/foo/foo.tsx", dependencies: ["src/foo/x.ts"] },
+ ];
+
+ it("forces a full Loki run when the Storybook preview imports a changed file", () => {
+ const plan = createTestPlan({
+ ...baseInput,
+ changedFiles: ["src/utils/colors.ts"],
+ loadFileDependencies: () => PREVIEW_DEPS,
+ e2eSpecFiles: { "e2e/test/scenarios/a.cy.spec.ts": ["src/bar/b.ts"] },
+ });
+
+ expect(plan.stats.loki_preview_touched).toBe(true);
+ expect(plan.stats.loki_stories_to_run_rules).toBe(LOKI_FILES.length);
+ expect(plan.loki_stories_to_run).toEqual(LOKI_FILES);
+ // Unit and e2e still narrow to what the usage graph reaches from the change.
+ expect(plan.fe_unit_specs_to_run).toEqual(["src/utils/utils.unit.spec.ts"]);
+ expect(plan.e2e_specs_to_run).toEqual([]);
+ });
+
+ it("narrows Loki when the Storybook preview does not import the changed file", () => {
+ const plan = createTestPlan({
+ ...baseInput,
+ changedFiles: ["src/foo/x.ts"],
+ loadFileDependencies: () => PREVIEW_DEPS,
+ });
+
+ expect(plan.stats.loki_preview_touched).toBe(false);
+ expect(plan.loki_stories_to_run).toEqual(["src/foo/Foo.stories.tsx"]);
+ });
+
+ it("runs all Loki stories when the cruise fails even if rules select fewer", () => {
+ const plan = createTestPlan({
+ ...baseInput,
+ changedFiles: ["src/foo/x.ts"],
+ loadFileDependencies: () => null,
+ e2eSpecFiles: { "e2e/test/scenarios/a.cy.spec.ts": ["src/bar/b.ts"] },
+ });
+
+ expect(plan.stats.loki_preview_touched).toBe(false);
+ expect(plan.loki_stories_to_run).toEqual(LOKI_FILES);
+ expect(plan.stats.loki_stories_to_run_rules).toBe(LOKI_FILES.length);
+ expect(plan.stats.loki_stories_to_run_usage).toBe(LOKI_FILES.length);
+ // Unit and E2E keep their rules-graph fallback.
+ expect(plan.fe_unit_specs_to_run).toEqual(UNIT_FILES.slice(0, 2));
+ expect(plan.e2e_specs_to_run).toEqual([]);
+ });
+
it("runs the full e2e suite when no coverage manifest is available", () => {
- const plan = createTestPlan({ ...baseInput, changedFiles: ["src/foo/x.ts"] });
+ const plan = createTestPlan({
+ ...baseInput,
+ changedFiles: ["src/foo/x.ts"],
+ });
expect(plan.stats.e2e_specs_to_run_rules).toBe(E2E_FILES.length);
expect(plan.stats.e2e_specs_to_run_usage).toBe(E2E_FILES.length);
diff --git a/.github/scripts/create-test-plan.ts b/.github/scripts/create-test-plan.ts
index c300b7833ac..6f9d5de6047 100644
--- a/.github/scripts/create-test-plan.ts
+++ b/.github/scripts/create-test-plan.ts
@@ -6,6 +6,7 @@ import { appendFileSync, existsSync, readFileSync } from "node:fs";
import micromatch from "micromatch";
+import { MAIN_APP_STORY_GLOBS } from "../../.storybook/story-files.cjs";
import { elements, rules } from "../../frontend/lint/module-boundaries.mjs";
import { type FileDependency, parseCruiseModules } from "./affected-modules";
@@ -22,11 +23,9 @@ const UNIT_GLOBS = [
"!**/*.leak.unit.spec.{js,jsx,ts,tsx}", // run only by jest.memory.conf.js
];
-const STORY_ROOTS = ["frontend", "enterprise/frontend"];
-const STORY_GLOBS = [
- "frontend/**/*.stories.{js,jsx,ts,tsx}",
- "enterprise/frontend/**/*.stories.{js,jsx,ts,tsx}",
-];
+const FRONTEND_ROOTS = ["frontend", "enterprise/frontend"];
+
+const LOKI_PREVIEW_FILE = ".storybook/preview.tsx";
// `git ls-files -- frontend enterprise/frontend` already prints just over a
// megabyte of paths, and node's default maxBuffer is exactly 1 MiB: past that
@@ -52,7 +51,7 @@ const csvToList = (csv: string | undefined) =>
.map((s) => s.trim())
.filter(Boolean);
-// Runs dependency-cruiser over the frontend sources and parses its edges.
+// Runs dependency-cruiser over the frontend sources and the Storybook preview and parses its edges.
// Null falls back to the rules graph, so a failed cruise never breaks the plan.
function loadFileDependencies(): FileDependency[] | null {
const output = "dependency-graph.json";
@@ -65,6 +64,7 @@ function loadFileDependencies(): FileDependency[] | null {
"depcruise",
"frontend/src",
"enterprise/frontend/src",
+ LOKI_PREVIEW_FILE,
"--config",
".dependency-cruiser.cjs",
"--output-type",
@@ -120,9 +120,10 @@ const testPlan = createTestPlan({
loadFileDependencies,
testFilesBySuite: {
unit: listFiles(UNIT_ROOTS, UNIT_GLOBS),
- loki: listFiles(STORY_ROOTS, STORY_GLOBS),
+ loki: listFiles(FRONTEND_ROOTS, MAIN_APP_STORY_GLOBS),
e2e: listSpecFiles(),
},
+ lokiPreviewFile: LOKI_PREVIEW_FILE,
e2eSpecFiles: readE2eSpecFiles(),
unitInfraTouched: process.env.UNIT_INFRA_TOUCHED === "true",
lokiInfraTouched: process.env.LOKI_INFRA_TOUCHED === "true",
@@ -130,7 +131,7 @@ const testPlan = createTestPlan({
sharedSourcesTouched: process.env.SHARED_SOURCES_TOUCHED === "true",
feFilesChanged: csvToList(process.env.FE_CHANGED_FILES).length,
beFilesChanged: csvToList(process.env.BE_CHANGED_FILES).length,
- feFilesTotal: listFiles(["frontend", "enterprise/frontend"], ["**"]).length,
+ feFilesTotal: listFiles(FRONTEND_ROOTS, ["**"]).length,
beFilesTotal: listFiles(["src", "enterprise/backend"], ["**"]).length,
});
diff --git a/.github/scripts/file-paths.unit.spec.ts b/.github/scripts/file-paths.unit.spec.ts
index 42df4da3756..a941df30e02 100644
--- a/.github/scripts/file-paths.unit.spec.ts
+++ b/.github/scripts/file-paths.unit.spec.ts
@@ -49,9 +49,9 @@ describe("file-paths.yaml", () => {
"%s skips a ratchets-only change and still follows the rest of .clj-kondo",
(name) => {
expect(matches(name, ".clj-kondo/ratchets.edn")).toBe(false);
- expect(
- matches(name, ".clj-kondo/config/modules/ratchets.edn"),
- ).toBe(false);
+ expect(matches(name, ".clj-kondo/config/modules/ratchets.edn")).toBe(
+ false,
+ );
expect(matches(name, ".clj-kondo/config.edn")).toBe(true);
expect(matches(name, ".clj-kondo/config/modules/config.edn")).toBe(true);
},
@@ -80,12 +80,15 @@ describe("file-paths.yaml", () => {
).toBe(true);
});
- it.each(["jest.config.js", "jest.base.conf.js", "jest.esm-packages.js"])(
- "runs CI script tests when %s changes",
- (path) => {
- expect(matches("ci_scripts", path)).toBe(true);
- },
- );
+ it.each([
+ "jest.config.js",
+ "jest.base.conf.js",
+ "jest.esm-packages.js",
+ ".storybook/story-files.cjs",
+ ".github/workflows/loki.yml",
+ ])("runs CI script tests when %s changes", (path) => {
+ expect(matches("ci_scripts", path)).toBe(true);
+ });
it("runs the ratchet check on the ratchets file", () => {
expect(matches("project_ratchet_checks", ".clj-kondo/ratchets.edn")).toBe(
@@ -98,4 +101,42 @@ describe("file-paths.yaml", () => {
),
).toBe(true);
});
+
+ it.each([
+ "loki.config.js",
+ ".storybook/story-files.cjs",
+ ".storybook/preview.tsx",
+ ".loki/reference/chrome.laptop-Button.png",
+ ".github/workflows/loki.yml",
+ ".github/workflows/run-tests.yml",
+ ".github/actions/prepare-frontend/action.yml",
+ "frontend/build/shared/rspack/css-config.js",
+ "frontend/test/__support__/custom-viz-fixtures/calendar-heatmap/index.js",
+ "patches/@loki+browser+0.35.0.patch",
+ "frontend/src/metabase/css/index.module.css",
+ "frontend/src/metabase/css/core/base.module.css",
+ "resources/frontend_client/app/fonts/Lato/lato-v16-latin-regular.woff2",
+ "resources/frontend_client/app/assets/img/no_results.svg",
+ "frontend/src/metabase/ui/components/icons/Icon/icons/warning.svg",
+ "enterprise/frontend/src/metabase-enterprise/google_drive/database-error.svg",
+ ])("runs all Loki stories when %s changes", (file) => {
+ expect(matches("frontend_loki_all", file)).toBe(true);
+ expect(matches("frontend_loki_infra", file)).toBe(true);
+ });
+
+ it.each([
+ "frontend/src/metabase/ui/Button.stories.tsx",
+ "frontend/src/metabase/ui/components/feedback/Alert/Alert.module.css",
+ "frontend/build/embedding-sdk/rspack.config.js",
+ ])("allows a change to %s to narrow the Loki run", (file) => {
+ expect(matches("frontend_loki_all", file)).toBe(true);
+ expect(matches("frontend_loki_infra", file)).toBe(false);
+ });
+
+ it.each([
+ ".github/workflows/frontend.yml",
+ "frontend/test/metabase/scenarios/Button.unit.spec.tsx",
+ ])("does not run Loki stories when %s changes", (file) => {
+ expect(matches("frontend_loki_all", file)).toBe(false);
+ });
});
diff --git a/.github/scripts/frontend-test-plan.unit.spec.ts b/.github/scripts/frontend-test-plan.unit.spec.ts
index ee34be24938..b2dc04bc629 100644
--- a/.github/scripts/frontend-test-plan.unit.spec.ts
+++ b/.github/scripts/frontend-test-plan.unit.spec.ts
@@ -16,7 +16,7 @@ import {
findEmptySelection,
findRunnerFailure,
} from "./check-unit-test-results";
-import { prepareUnitTestSelection } from "./prepare-unit-test-selection";
+import { prepareTestSelection } from "./prepare-test-selection";
type Step = { name: string; run?: string; "continue-on-error"?: boolean };
// js-yaml returns an untyped value, and this repository workflow defines the job's steps.
@@ -69,79 +69,6 @@ describe("frontend test plan handoff", () => {
writeFileSync(join(dir, "test-plan/test-plan.json"), JSON.stringify(value));
}
- it.each([{ files: ["one.spec.cjs"] }, { files: [] }])(
- "preserves an explicit selection: $files",
- ({ files }) => {
- writePlan(plan(files));
- prepareUnitTestSelection(env);
- expect(readFileSync(join(dir, "output"), "utf8")).toContain(
- "paths-file=",
- );
- expect(
- JSON.parse(readFileSync(join(dir, "unit-specs.json"), "utf8")),
- ).toEqual(files);
- },
- );
-
- it("writes the selection when invoked through Bun in CI", () => {
- writePlan(plan(["one.spec.cjs"]));
- const result = spawnSync(
- "bun",
- [resolve(__dirname, "prepare-unit-test-selection.ts")],
- { env, encoding: "utf8" },
- );
- expect(result.status === 0 ? "" : result.stderr).toBe("");
- expect(result.status).toBe(0);
- expect(readFileSync(join(dir, "output"), "utf8")).toBe(
- `paths-file=${join(dir, "unit-specs.json")}\n`,
- );
- expect(
- JSON.parse(readFileSync(join(dir, "unit-specs.json"), "utf8")),
- ).toEqual(["one.spec.cjs"]);
- });
-
- it("bypasses the filter for a full selection", () => {
- writePlan(plan(["one.spec.cjs", "two.spec.cjs"]));
- prepareUnitTestSelection(env);
- expect(readFileSync(join(dir, "output"), "utf8")).toBe("");
- });
-
- it.each([
- null,
- {},
- { ...plan([]), fe_unit_specs_to_run: "[]" },
- { ...plan([]), fe_unit_specs_to_run: [null] },
- { ...plan([]), fe_unit_specs_to_run: ["one.spec.cjs"] },
- plan(["one.spec.cjs"], 0),
- ])("runs in full for an invalid plan: %j", (value) => {
- writePlan(value);
- prepareUnitTestSelection(env);
- expect(console.warn).toHaveBeenCalledWith(
- expect.stringContaining("::warning::"),
- );
- expect(readFileSync(join(dir, "output"), "utf8")).toBe("");
- });
-
- it.each(["missing", "truncated", "download failed"])(
- "runs in full when the artifact is %s",
- (scenario) => {
- if (scenario === "truncated") {
- writeFileSync(
- join(dir, "test-plan/test-plan.json"),
- '{"fe_unit_specs_to_run":[',
- );
- } else if (scenario === "download failed") {
- writePlan(plan([])); // Even a partial download must not be trusted.
- env.PLAN_DOWNLOADED = "failure";
- }
- prepareUnitTestSelection(env);
- expect(console.warn).toHaveBeenCalledWith(
- expect.stringContaining("::warning::"),
- );
- expect(readFileSync(join(dir, "output"), "utf8")).toBe("");
- },
- );
-
function jestCommand(useFilter: boolean) {
return [
process.execPath,
@@ -251,7 +178,7 @@ describe("frontend test plan handoff", () => {
"loads the TypeScript filter natively with an empty selection: %s",
(empty) => {
writePlan(plan(empty ? [] : [join(dir, "pass.spec.cjs")]));
- prepareUnitTestSelection(env);
+ prepareTestSelection("unit", env);
env.JEST_TEST_PATHS_FILE = join(dir, "unit-specs.json");
const result = runJest(
{
diff --git a/.github/scripts/prepare-test-selection.ts b/.github/scripts/prepare-test-selection.ts
new file mode 100644
index 00000000000..4e31f6d5a18
--- /dev/null
+++ b/.github/scripts/prepare-test-selection.ts
@@ -0,0 +1,122 @@
+import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import type { TestPlan, TestPlanStats } from "./affected-tests";
+
+type NumberStat = {
+ [K in keyof TestPlanStats]: TestPlanStats[K] extends number ? K : never;
+}[keyof TestPlanStats];
+
+type Suite = {
+ name: string;
+ filesKey: Exclude<keyof TestPlan, "stats">;
+ totalKey: NumberStat;
+ selectedKey: NumberStat;
+ pathsFile: string;
+};
+
+export const SUITES = {
+ unit: {
+ name: "unit",
+ filesKey: "fe_unit_specs_to_run",
+ totalKey: "unit_specs_all",
+ selectedKey: "unit_specs_to_run_usage",
+ pathsFile: "unit-specs.json",
+ },
+ loki: {
+ name: "Loki",
+ filesKey: "loki_stories_to_run",
+ totalKey: "loki_stories_all",
+ selectedKey: "loki_stories_to_run_usage",
+ pathsFile: "loki-stories.json",
+ },
+ e2e: {
+ name: "E2E",
+ filesKey: "e2e_specs_to_run",
+ totalKey: "e2e_specs_all",
+ selectedKey: "e2e_specs_to_run_usage",
+ pathsFile: "e2e-specs.json",
+ },
+} satisfies Record<string, Suite>;
+
+function isSuiteName(name: string): name is keyof typeof SUITES {
+ return Object.hasOwn(SUITES, name);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null;
+}
+
+function isStringArray(value: unknown): value is string[] {
+ return (
+ Array.isArray(value) && value.every((item) => typeof item === "string")
+ );
+}
+
+function isCount(value: unknown): value is number {
+ return typeof value === "number" && Number.isInteger(value) && value >= 0;
+}
+
+// null means a full run, and [] is an explicit selection of no tests.
+function readSelection(planFile: string, suite: Suite): string[] | null {
+ const plan: unknown = JSON.parse(readFileSync(planFile, "utf8"));
+ const files = isRecord(plan) ? plan[suite.filesKey] : undefined;
+ const stats = isRecord(plan) && isRecord(plan.stats) ? plan.stats : {};
+ const total = stats[suite.totalKey];
+ if (
+ !isStringArray(files) ||
+ files.some((file) => file.length === 0) ||
+ !isCount(total) ||
+ files.length !== stats[suite.selectedKey] ||
+ files.length > total
+ ) {
+ throw new Error(`Invalid ${suite.name} test selection`);
+ }
+ return files.length === total ? null : files;
+}
+
+export function prepareTestSelection(
+ suiteName: keyof typeof SUITES,
+ env: NodeJS.ProcessEnv = process.env,
+) {
+ const suite: Suite = SUITES[suiteName];
+ const { RUNNER_TEMP, GITHUB_OUTPUT, PLAN_DOWNLOADED } = env;
+ if (!RUNNER_TEMP || !GITHUB_OUTPUT) {
+ throw new Error("RUNNER_TEMP and GITHUB_OUTPUT are required");
+ }
+
+ let files: string[] | null;
+ try {
+ if (PLAN_DOWNLOADED !== "success") {
+ throw new Error("Test plan download failed");
+ }
+ files = readSelection(join(RUNNER_TEMP, "test-plan/test-plan.json"), suite);
+ } catch (error) {
+ console.warn(
+ `::warning::Test plan unavailable or invalid, running the full ${suite.name} suite: ${error}`,
+ );
+ return;
+ }
+
+ if (files === null) {
+ appendFileSync(GITHUB_OUTPUT, "selection=full\n");
+ return;
+ }
+
+ const pathsFile = join(RUNNER_TEMP, suite.pathsFile);
+ writeFileSync(pathsFile, JSON.stringify(files));
+ appendFileSync(
+ GITHUB_OUTPUT,
+ `paths-file=${pathsFile}\nselection=${files.length === 0 ? "empty" : "narrowed"}\n`,
+ );
+}
+
+if (require.main === module) {
+ const suiteName = process.argv[2] ?? "";
+ if (!isSuiteName(suiteName)) {
+ throw new Error(
+ `Unknown test suite "${suiteName}", expected one of: ${Object.keys(SUITES).join(", ")}`,
+ );
+ }
+ prepareTestSelection(suiteName);
+}
diff --git a/.github/scripts/prepare-test-selection.unit.spec.ts b/.github/scripts/prepare-test-selection.unit.spec.ts
new file mode 100644
index 00000000000..e705d9af3bd
--- /dev/null
+++ b/.github/scripts/prepare-test-selection.unit.spec.ts
@@ -0,0 +1,268 @@
+import { spawnSync } from "node:child_process";
+import {
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+import { load } from "js-yaml";
+
+import { SUITES, prepareTestSelection } from "./prepare-test-selection";
+
+type Suite = (typeof SUITES)[keyof typeof SUITES];
+
+type Step = {
+ id?: string;
+ if?: string;
+ uses?: string;
+ run?: string;
+ env?: Record<string, string>;
+ with?: Record<string, string | boolean>;
+};
+
+const SCRIPT = resolve(__dirname, "prepare-test-selection.ts");
+const SUITE_NAMES = ["unit", "loki", "e2e"] satisfies (keyof typeof SUITES)[];
+
+function loadWorkflow(file: string) {
+ // js-yaml returns an untyped value, and these repository workflows define jobs with steps.
+ return load(
+ readFileSync(resolve(__dirname, "../workflows", file), "utf8"),
+ ) as {
+ jobs: Record<
+ string,
+ { if?: string; outputs?: Record<string, string>; steps?: Step[] }
+ >;
+ };
+}
+
+function loadAction() {
+ // js-yaml returns an untyped value, and this repository action defines outputs and steps.
+ return load(
+ readFileSync(
+ resolve(__dirname, "../actions/prepare-test-selection/action.yml"),
+ "utf8",
+ ),
+ ) as {
+ outputs: Record<string, { value: string }>;
+ runs: { steps: Step[] };
+ };
+}
+
+describe("prepareTestSelection", () => {
+ const { loki } = SUITES;
+ let dir: string;
+ let env: NodeJS.ProcessEnv;
+
+ beforeEach(() => {
+ jest.spyOn(console, "warn").mockImplementation(() => {});
+ dir = mkdtempSync(join(tmpdir(), "test-selection-"));
+ mkdirSync(join(dir, "test-plan"));
+ writeFileSync(join(dir, "output"), "");
+ env = {
+ ...process.env,
+ RUNNER_TEMP: dir,
+ GITHUB_OUTPUT: join(dir, "output"),
+ PLAN_DOWNLOADED: "success",
+ };
+ });
+
+ afterEach(() => {
+ jest.restoreAllMocks();
+ rmSync(dir, { recursive: true, force: true });
+ });
+
+ function plan(files: unknown, total = 2, suite: Suite = loki) {
+ return {
+ [suite.filesKey]: files,
+ stats: {
+ [suite.totalKey]: total,
+ [suite.selectedKey]: Array.isArray(files) ? files.length : 0,
+ },
+ };
+ }
+
+ function writePlan(value: unknown) {
+ writeFileSync(join(dir, "test-plan/test-plan.json"), JSON.stringify(value));
+ }
+
+ const output = () => readFileSync(join(dir, "output"), "utf8");
+
+ it.each([{ files: ["one.spec.cjs"] }, { files: [] }])(
+ "preserves an explicit selection: $files",
+ ({ files }) => {
+ writePlan(plan(files));
+ prepareTestSelection("loki", env);
+ expect(output()).toBe(
+ `paths-file=${join(dir, loki.pathsFile)}\nselection=${files.length > 0 ? "narrowed" : "empty"}\n`,
+ );
+ expect(
+ JSON.parse(readFileSync(join(dir, loki.pathsFile), "utf8")),
+ ).toEqual(files);
+ },
+ );
+
+ it("writes the selection when invoked through Bun in CI", () => {
+ writePlan(plan(["one.spec.cjs"]));
+ const result = spawnSync("bun", [SCRIPT, "loki"], {
+ env,
+ encoding: "utf8",
+ });
+ expect(result.status === 0 ? "" : result.stderr).toBe("");
+ expect(result.status).toBe(0);
+ expect(output()).toBe(
+ `paths-file=${join(dir, loki.pathsFile)}\nselection=narrowed\n`,
+ );
+ });
+
+ it("writes no paths file for a full selection", () => {
+ writePlan(plan(["one.spec.cjs", "two.spec.cjs"]));
+ prepareTestSelection("loki", env);
+ expect(output()).toBe("selection=full\n");
+ });
+
+ it.each([
+ null,
+ {},
+ plan("[]"),
+ plan([null]),
+ { ...plan([]), [loki.filesKey]: ["one.spec.cjs"] },
+ plan([""]),
+ plan(["one.spec.cjs"], 0),
+ ])("runs in full for an invalid plan: %j", (value) => {
+ writePlan(value);
+ prepareTestSelection("loki", env);
+ expect(console.warn).toHaveBeenCalledWith(
+ expect.stringContaining("::warning::"),
+ );
+ expect(output()).toBe("");
+ });
+
+ it.each([
+ { scenario: "the test plan is missing", setup: () => {} },
+ {
+ scenario: "the test plan is truncated",
+ setup: () => {
+ writeFileSync(
+ join(dir, "test-plan/test-plan.json"),
+ `{"${loki.filesKey}":[`,
+ );
+ },
+ },
+ {
+ scenario: "the download failed",
+ setup: () => {
+ writePlan(plan([])); // Even a partial download must not be trusted.
+ env.PLAN_DOWNLOADED = "failure";
+ },
+ },
+ ])("runs in full when $scenario", ({ setup }) => {
+ setup();
+ prepareTestSelection("loki", env);
+ expect(console.warn).toHaveBeenCalledWith(
+ expect.stringContaining("::warning::"),
+ );
+ expect(output()).toBe("");
+ });
+
+ it.each(SUITE_NAMES)(
+ "reads the %s selection from its own plan keys",
+ (suiteName) => {
+ const suite = SUITES[suiteName];
+ writePlan(plan(["one.spec.cjs"], 2, suite));
+ prepareTestSelection(suiteName, env);
+ expect(output()).toBe(
+ `paths-file=${join(dir, suite.pathsFile)}\nselection=narrowed\n`,
+ );
+ expect(
+ JSON.parse(readFileSync(join(dir, suite.pathsFile), "utf8")),
+ ).toEqual(["one.spec.cjs"]);
+ },
+ );
+
+ it("ignores the other suites' selections", () => {
+ writePlan({
+ ...plan(["one.spec.cjs", "two.spec.cjs"]),
+ [SUITES.unit.filesKey]: [],
+ [SUITES.e2e.filesKey]: [],
+ });
+ prepareTestSelection("loki", env);
+ expect(console.warn).not.toHaveBeenCalled();
+ expect(output()).toBe("selection=full\n");
+ });
+
+ it("rejects an unknown suite", () => {
+ const result = spawnSync("bun", [SCRIPT, "toString"], {
+ env,
+ encoding: "utf8",
+ });
+ expect(result.status).not.toBe(0);
+ expect(result.stderr).toContain('Unknown test suite "toString"');
+ });
+});
+
+describe("test selection workflow steps", () => {
+ const ACTION = "./.github/actions/prepare-test-selection";
+ const CONSUMERS: Record<string, { suite: keyof typeof SUITES }> = {
+ "frontend.yml": { suite: "unit" },
+ "loki.yml": { suite: "loki" },
+ };
+
+ it("skips Loki visual tests only for an empty story selection", () => {
+ const { jobs } = loadWorkflow("loki.yml");
+ expect(jobs["story-selection"].outputs?.selection).toBe(
+ "${{ steps.loki-selection.outputs.selection }}",
+ );
+ expect(jobs["visual-test"].if).toContain(
+ "needs.story-selection.outputs.selection != 'empty'",
+ );
+ expect(jobs["visual-test"].if).not.toMatch(/outputs\.[\w-]+ *[!=]= *'\d+'/);
+ });
+
+ it.each(Object.entries(CONSUMERS))(
+ "selects tests through the composite action in %s",
+ (file, { suite, ...inputs }) => {
+ const steps = Object.values(loadWorkflow(file).jobs).flatMap(
+ ({ steps = [] }) => steps.filter((step) => step.uses === ACTION),
+ );
+
+ expect(steps.length).toBeGreaterThan(0);
+ for (const step of steps) {
+ expect(step.with).toEqual({
+ suite,
+ "test-plan-artifact-id": expect.stringMatching(/^\$\{\{ .+ \}\}$/),
+ ...inputs,
+ });
+ }
+ },
+ );
+
+ it("downloads the test plan before preparing the selection", () => {
+ const { outputs, runs } = loadAction();
+ const [download, prepare] = runs.steps;
+
+ expect(download).toMatchObject({
+ uses: expect.stringMatching(/^actions\/download-artifact@/),
+ "continue-on-error": "${{ inputs.require-download != 'true' }}",
+ with: {
+ "artifact-ids": "${{ inputs.test-plan-artifact-id }}",
+ path: "${{ runner.temp }}/test-plan",
+ },
+ });
+ expect(prepare).toMatchObject({
+ env: { PLAN_DOWNLOADED: `\${{ steps.${download.id}.outcome }}` },
+ run: expect.stringContaining(
+ 'prepare-test-selection.ts "${{ inputs.suite }}"',
+ ),
+ });
+ expect(outputs).toMatchObject({
+ "paths-file": {
+ value: `\${{ steps.${prepare.id}.outputs.paths-file }}`,
+ },
+ selection: { value: `\${{ steps.${prepare.id}.outputs.selection }}` },
+ });
+ });
+});
diff --git a/.github/scripts/prepare-unit-test-selection.ts b/.github/scripts/prepare-unit-test-selection.ts
deleted file mode 100644
index 74873d2e7d6..00000000000
--- a/.github/scripts/prepare-unit-test-selection.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-
-// null means a full run, and [] is an explicit selection of no tests.
-function readSelection(planFile: string): string[] | null {
- const { fe_unit_specs_to_run: files, stats } = JSON.parse(
- readFileSync(planFile, "utf8"),
- );
- if (
- !Array.isArray(files) ||
- !files.every((file) => typeof file === "string" && file.length > 0) ||
- !Number.isInteger(stats?.unit_specs_all) ||
- stats.unit_specs_all < 0 ||
- files.length !== stats.unit_specs_to_run_usage ||
- files.length > stats.unit_specs_all
- ) {
- throw new Error("Invalid unit test selection");
- }
- return files.length === stats.unit_specs_all ? null : files;
-}
-
-export function prepareUnitTestSelection(env: NodeJS.ProcessEnv = process.env) {
- const { RUNNER_TEMP, GITHUB_OUTPUT, PLAN_DOWNLOADED } = env;
- if (!RUNNER_TEMP || !GITHUB_OUTPUT) {
- throw new Error("RUNNER_TEMP and GITHUB_OUTPUT are required");
- }
-
- let files: string[] | null;
- try {
- if (PLAN_DOWNLOADED !== "success") {
- throw new Error("Test plan download failed");
- }
- files = readSelection(join(RUNNER_TEMP, "test-plan/test-plan.json"));
- } catch {
- console.warn(
- "::warning::Test plan unavailable or invalid; running the full unit suite.",
- );
- return;
- }
-
- // Full selections use Jest's normal discovery, without the filter.
- if (files === null) {
- return;
- }
-
- const pathsFile = join(RUNNER_TEMP, "unit-specs.json");
- writeFileSync(pathsFile, JSON.stringify(files));
- appendFileSync(GITHUB_OUTPUT, `paths-file=${pathsFile}\n`);
-}
-
-if (require.main === module) {
- prepareUnitTestSelection();
-}
diff --git a/.github/scripts/story-files.unit.spec.ts b/.github/scripts/story-files.unit.spec.ts
new file mode 100644
index 00000000000..1ca15e4b522
--- /dev/null
+++ b/.github/scripts/story-files.unit.spec.ts
@@ -0,0 +1,226 @@
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+import micromatch, { isMatch } from "micromatch";
+import { normalizeStories } from "storybook/internal/common";
+
+import {
+ MAIN_APP_STORY_GLOBS,
+ getStories,
+ hasExplicitTitle,
+} from "../../.storybook/story-files.cjs";
+
+const ROOT = resolve(__dirname, "../..");
+
+const tracked = execFileSync(
+ "git",
+ ["ls-files", "-z", "--", "frontend", "enterprise/frontend"],
+ { cwd: ROOT, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 },
+)
+ .split("\0")
+ .filter(Boolean);
+const planned = micromatch(tracked, MAIN_APP_STORY_GLOBS, { dot: true }).sort();
+
+describe("getStories", () => {
+ let tempDir: string;
+ let pathsFile: string;
+
+ beforeEach(() => {
+ tempDir = mkdtempSync(join(tmpdir(), "story-files-"));
+ pathsFile = join(tempDir, "stories.json");
+ });
+
+ afterEach(() => {
+ rmSync(tempDir, { recursive: true, force: true });
+ });
+
+ it("loads the main app stories and docs when there is no plan", () => {
+ const patterns = getStories({ pathsFile: "", filter: "" });
+
+ expect(
+ isMatch("../frontend/src/metabase/ui/Button.stories.tsx", patterns),
+ ).toBe(true);
+ expect(isMatch("../frontend/src/metabase/ui/Button.mdx", patterns)).toBe(
+ true,
+ );
+ expect(
+ isMatch(
+ "../frontend/src/embedding-sdk-shared/Button.stories.tsx",
+ patterns,
+ ),
+ ).toBe(false);
+ });
+
+ it("builds only the files in the JSON plan", () => {
+ const files = planned.slice(0, 2);
+ writeFileSync(pathsFile, JSON.stringify(files));
+
+ expect(getStories({ pathsFile })).toEqual(
+ files.map((file) => `../${file}`),
+ );
+ });
+
+ it("builds a planned story with an explicit title", () => {
+ const story = join(tempDir, "Titled.stories.tsx");
+ writeFileSync(
+ story,
+ 'const meta = { title: "Components/Titled" };\nexport default meta;\n',
+ );
+ writeFileSync(pathsFile, JSON.stringify([story]));
+
+ expect(getStories({ pathsFile })).toEqual([`../${story}`]);
+ });
+
+ it("rejects a planned story without an explicit title", () => {
+ const story = join(tempDir, "Untitled.stories.tsx");
+ writeFileSync(story, "export default { component: Untitled };\n");
+ writeFileSync(pathsFile, JSON.stringify([story]));
+
+ expect(() => getStories({ pathsFile })).toThrow(story);
+ });
+
+ it("rejects an untitled Alert story even though its args have a title", () => {
+ const source = readFileSync(
+ join(
+ ROOT,
+ "frontend/src/metabase/ui/components/feedback/Alert/Alert.stories.tsx",
+ ),
+ "utf8",
+ ).replace(' title: "Components/Feedback/Alert",\n', "");
+ const story = join(tempDir, "Alert.stories.tsx");
+ writeFileSync(story, source);
+ writeFileSync(pathsFile, JSON.stringify([story]));
+
+ expect(() => getStories({ pathsFile })).toThrow(story);
+ });
+
+ it("rejects an empty plan even when the CSV filter is set", () => {
+ writeFileSync(pathsFile, "[]");
+
+ expect(() =>
+ getStories({ pathsFile, filter: "frontend/src/Unselected.stories.tsx" }),
+ ).toThrow("No stories selected");
+ });
+
+ it("preserves the stress test's CSV filter", () => {
+ expect(
+ getStories({
+ filter: "frontend/One.stories.tsx,frontend/Two.stories.tsx",
+ }),
+ ).toEqual(["../frontend/One.stories.tsx", "../frontend/Two.stories.tsx"]);
+ });
+
+ it.each(["{", "null", '""', "{}", "[null]"])(
+ "rejects an invalid plan: %s",
+ (contents) => {
+ writeFileSync(pathsFile, contents);
+
+ expect(() => getStories({ pathsFile })).toThrow();
+ },
+ );
+});
+
+describe("hasExplicitTitle", () => {
+ it.each([
+ {
+ form: "double quotes",
+ source: 'export default {\n title: "Components/Button",\n};\n',
+ },
+ {
+ form: "single quotes",
+ source: "export default {\n title: 'Components/Button',\n};\n",
+ },
+ {
+ form: "a title constant",
+ source: 'const title = "Components/Button"; export default { title };',
+ },
+ {
+ form: "a meta constant",
+ source:
+ 'const meta: Meta<typeof Button> = {\n title: "Components/Button",\n};\n\nexport default meta;\n',
+ },
+ {
+ form: "a satisfies expression",
+ source:
+ 'const meta = { title: "Components/Button" } satisfies Meta<typeof Button>; export default meta;',
+ },
+ ])("accepts a title in $form", ({ source }) => {
+ expect(hasExplicitTitle(source)).toBe(true);
+ });
+
+ it.each([
+ {
+ form: "no title",
+ source: "export default {\n component: Button,\n};\n",
+ },
+ {
+ form: "only a title control",
+ source:
+ 'const argTypes = {\n title: {\n control: { type: "text" },\n },\n};\n\nexport default { component: Button, argTypes };\n',
+ },
+ {
+ form: "only a title variable",
+ source:
+ "export default {\n component: Button,\n args: {\n title: defaultTitle,\n },\n};\n",
+ },
+ {
+ form: "only a string-valued args title",
+ source: 'export default { args: { title: "Button label" } };',
+ },
+ {
+ form: "only another object's title",
+ source:
+ 'const other = { title: "Other" }; export default { component: Button };',
+ },
+ {
+ form: "only a commented title",
+ source:
+ 'export default {\n // title: "Components/Button",\n component: Button };',
+ },
+ {
+ form: "an empty title",
+ source: 'export default { title: "" };',
+ },
+ {
+ form: "a template literal unsupported by Storybook",
+ source: "export default { title: `Components/Button` };",
+ },
+ ])("rejects $form", ({ source }) => {
+ expect(hasExplicitTitle(source)).toBe(false);
+ });
+});
+
+describe("MAIN_APP_STORY_GLOBS", () => {
+ it("matches the same tracked files as Storybook's own story matcher", () => {
+ const specifiers = normalizeStories(getStories(), {
+ configDir: join(ROOT, ".storybook"),
+ workingDir: ROOT,
+ });
+ const built = tracked
+ .filter(
+ (file) =>
+ !file.endsWith(".mdx") &&
+ specifiers.some((specifier) =>
+ specifier.importPathMatcher.test(`./${file}`),
+ ),
+ )
+ .sort();
+
+ expect(planned.length).toBeGreaterThan(0);
+ expect(planned).toEqual(built);
+ });
+
+ it("matches no docs entries", () => {
+ expect(planned.filter((file) => file.endsWith(".mdx"))).toEqual([]);
+ });
+
+ it("matches only stories with an explicit title", () => {
+ const untitled = planned.filter(
+ (file) => !hasExplicitTitle(readFileSync(join(ROOT, file), "utf8")),
+ );
+
+ expect(untitled).toEqual([]);
+ });
+});
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
index 7e81ed45b62..acd18521016 100644
--- a/.github/workflows/frontend.yml
+++ b/.github/workflows/frontend.yml
@@ -118,20 +118,13 @@ jobs:
# Resolve JOB_ID before the long test step so the ci-conductor reporter
# below can link failures to this exact job. Best-effort.
uses: ./.github/actions/resolve-job-id
- - name: Download test plan
- id: test-plan
- if: ${{ inputs.test-plan-artifact-id != '' }}
- continue-on-error: true
- uses: actions/download-artifact@v8
- with:
- artifact-ids: ${{ inputs.test-plan-artifact-id }}
- path: ${{ runner.temp }}/test-plan
- name: Prepare unit test selection
id: unit-selection
if: ${{ inputs.test-plan-artifact-id != '' }}
- env:
- PLAN_DOWNLOADED: ${{ steps.test-plan.outcome }}
- run: bun .github/scripts/prepare-unit-test-selection.ts
+ uses: ./.github/actions/prepare-test-selection
+ with:
+ suite: unit
+ test-plan-artifact-id: ${{ inputs.test-plan-artifact-id }}
- name: Run frontend unit tests
id: fe-tests
env:
diff --git a/.github/workflows/loki.yml b/.github/workflows/loki.yml
index 7d8b6502bcf..36824a23bee 100644
--- a/.github/workflows/loki.yml
+++ b/.github/workflows/loki.yml
@@ -4,16 +4,51 @@ on:
workflow_call:
inputs:
skip:
+ description: "Skip the Loki visual tests."
type: boolean
default: false
+ test-plan-artifact-id:
+ description: "ID of this run's test-plan.json artifact, or '' to run the full suite."
+ type: string
+ default: ""
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}-loki
cancel-in-progress: true
jobs:
+ story-selection:
+ if: ${{ !inputs.skip && inputs.test-plan-artifact-id != '' }}
+ runs-on: ${{ vars.SLIM_RUNNER_KEY }}
+ timeout-minutes: 5
+ outputs:
+ selection: ${{ steps.loki-selection.outputs.selection }}
+ test-plan-artifact-id: ${{ steps.loki-selection.outputs.paths-file && inputs.test-plan-artifact-id || '' }}
+ steps:
+ - name: Checkout Repository
+ uses: actions/checkout@v6
+ with:
+ sparse-checkout: |
+ .github
+ /.nvmrc
+ /package.json
+ sparse-checkout-cone-mode: false
+
+ - name: Prepare Node + Bun environment
+ uses: ./.github/actions/prepare-node-bun
+
+ - name: Prepare Loki story selection
+ id: loki-selection
+ uses: ./.github/actions/prepare-test-selection
+ with:
+ suite: loki
+ test-plan-artifact-id: ${{ inputs.test-plan-artifact-id }}
+
visual-test:
- if: ${{ !inputs.skip }}
+ needs: story-selection
+ # A missing job output compares equal to '0' in GitHub expressions,
+ # so an empty selection is marked with a non-numeric value.
+ if: ${{ !cancelled() && !inputs.skip && needs.story-selection.outputs.selection != 'empty' }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
timeout-minutes: 30
services:
@@ -41,8 +76,22 @@ jobs:
- name: Compile CLJS
run: NODE_ENV=development bun run build-pure:cljs
+ - name: Prepare Loki story selection
+ id: loki-selection
+ if: ${{ needs.story-selection.outputs.test-plan-artifact-id != '' }}
+ uses: ./.github/actions/prepare-test-selection
+ with:
+ suite: loki
+ test-plan-artifact-id: ${{ needs.story-selection.outputs.test-plan-artifact-id }}
+
+ - name: Build Storybook
+ env:
+ STORYBOOK_STORY_PATHS_FILE: ${{ steps.loki-selection.outputs.paths-file }}
+ run: bun run build-storybook
+
- name: Run Loki Visual Tests
- run: bun run test-visual:loki:ci
+ # Loki drops stories with parameters.loki.skip, and exits non-zero when that leaves the selection empty.
+ run: bun run test-visual:loki --reactUri file:./storybook-static --verboseRenderer ${{ steps.loki-selection.outputs.paths-file && '--passWithNoStories --requireReference' || '' }}
- name: Generate Visual Report on Failure
if: failure()
@@ -57,3 +106,37 @@ jobs:
include-hidden-files: true
path: .loki/
if-no-files-found: ignore
+
+ loki-tests-result:
+ needs: [story-selection, visual-test]
+ if: always() && !cancelled()
+ runs-on: ${{ vars.SLIM_RUNNER_KEY }}
+ timeout-minutes: 5
+ steps:
+ - name: Check Loki job status
+ uses: actions/github-script@v9
+ env:
+ needs: ${{ toJson(needs) }}
+ with:
+ script: | # js
+ const needs = JSON.parse(process.env.needs);
+ const jobs = Object.entries(needs).map(
+ ([jobName, jobValues]) => ({
+ name: jobName,
+ result: jobValues.result
+ }));
+
+ // are all jobs skipped or successful?
+ if (jobs.every(job => (job.result === 'skipped' || job.result === 'success'))) {
+ console.log('All jobs are skipped or successful');
+ process.exit(0);
+ }
+
+ // otherwise, something failed
+ console.log('Some Loki jobs failed');
+ jobs.forEach((job) => {
+ if (job.result !== 'success') {
+ console.log(`${job.name} - ${job.result}`);
+ }
+ });
+ process.exit(1);
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index f1a344627c7..3eae2d71e6a 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -57,11 +57,10 @@ jobs:
build_scripts: ${{ steps.changes.outputs.build_scripts }}
embedder_plugin: ${{ steps.changes.outputs.embedder_plugin }}
frontend_all: ${{ steps.changes.outputs.frontend_all }}
- frontend_ci: ${{ steps.changes.outputs.frontend_ci }}
frontend_sources: ${{ steps.changes.outputs.frontend_sources }}
frontend_unit_infra: ${{ steps.changes.outputs.frontend_unit_infra }}
- frontend_loki_ci: ${{ steps.changes.outputs.frontend_loki_ci }}
frontend_loki_infra: ${{ steps.changes.outputs.frontend_loki_infra }}
+ frontend_loki_all: ${{ steps.changes.outputs.frontend_loki_all }}
shared_sources: ${{ steps.changes.outputs.shared_sources }}
documentation: ${{ steps.changes.outputs.documentation }}
embedding_documentation: ${{ steps.changes.outputs.embedding_documentation }}
@@ -320,7 +319,7 @@ jobs:
needs.files-changed.outputs.ci_scripts != 'true' }}
loki-tests:
- needs: [should-run, files-changed]
+ needs: [should-run, files-changed, create-test-plan]
if: |
!cancelled() &&
(needs.should-run.outputs.verdict == 'force-run' ||
@@ -329,9 +328,9 @@ jobs:
with:
skip: >-
${{ needs.should-run.outputs.verdict != 'force-run' &&
- needs.files-changed.outputs.frontend_ci != 'true' &&
- needs.files-changed.outputs.frontend_sources != 'true' &&
- needs.files-changed.outputs.frontend_loki_ci != 'true' }}
+ needs.files-changed.outputs.frontend_loki_all != 'true' }}
+ # A skipped/failed planner supplies no artifact, so the suite runs in full.
+ test-plan-artifact-id: ${{ needs.create-test-plan.result == 'success' && needs.create-test-plan.outputs.artifact-id || '' }}
e2e-tests:
needs: [should-run, files-changed, uberjar]
diff --git a/.storybook/main.ts b/.storybook/main.ts
index 751a1821c38..2b373ecb66d 100644
--- a/.storybook/main.ts
+++ b/.storybook/main.ts
@@ -12,21 +12,16 @@ const {
} = require("../frontend/build/shared/rspack/side-effect-free-modules");
const appConfig = require("../rspack.main.config.js");
-const mainAppStories = [
- "../frontend/**/*.mdx",
- "../frontend/*/!(embedding-sdk-bundle|embedding-sdk-shared)/**/*.stories.@(ts|tsx)",
- "../enterprise/frontend/*/!(embedding-sdk-ee|embedding-sdk-package)/**/*.stories.@(ts|tsx)",
-];
-
-// Allow filtering to specific story files via env var (used by stress tests)
-// STORYBOOK_STORIES_FILTER: comma-separated file paths relative to repo root
-// e.g. "frontend/src/.../Button.stories.tsx,frontend/src/.../Alert.stories.tsx"
-const stories = process.env.STORYBOOK_STORIES_FILTER
- ? process.env.STORYBOOK_STORIES_FILTER.split(",").map((f) => `../${f}`)
- : mainAppStories;
+const { getStories } = require("./story-files.cjs");
const config: StorybookConfig = {
- stories,
+ stories: getStories({
+ pathsFile: process.env.STORYBOOK_STORY_PATHS_FILE,
+ // Allow filtering to specific story files via env var (used by stress tests)
+ // STORYBOOK_STORIES_FILTER: comma-separated file paths relative to repo root
+ // e.g. "frontend/src/.../Button.stories.tsx,frontend/src/.../Alert.stories.tsx"
+ filter: process.env.STORYBOOK_STORIES_FILTER,
+ }),
staticDirs: [
"../resources/frontend_client",
"./msw-public",
diff --git a/.storybook/story-files.cjs b/.storybook/story-files.cjs
new file mode 100644
index 00000000000..9594988b203
--- /dev/null
+++ b/.storybook/story-files.cjs
@@ -0,0 +1,54 @@
+const { readFileSync } = require("node:fs");
+const { resolve } = require("node:path");
+
+const ROOT = resolve(__dirname, "..");
+
+const MAIN_APP_STORY_GLOBS = [
+ "frontend/*/!(embedding-sdk-bundle|embedding-sdk-shared)/**/*.stories.@(ts|tsx)",
+ "enterprise/frontend/*/!(embedding-sdk-ee|embedding-sdk-package)/**/*.stories.@(ts|tsx)",
+];
+
+// Storybook autotitles a story from its path relative to the entry's directory,
+// so an untitled story gets a different title in a narrowed build and Loki writes a fresh reference for it.
+function hasExplicitTitle(source) {
+ // Only load the parser when validating a build; the planner also imports this
+ // module for its story globs.
+ const { loadCsf } = require("storybook/internal/csf-tools");
+ try {
+ const { meta } = loadCsf(source, { makeTitle: (title) => title }).parse();
+ return typeof meta?.title === "string" && meta.title.length > 0;
+ } catch {
+ return false;
+ }
+}
+
+function getStories({ pathsFile, filter } = {}) {
+ if (pathsFile) {
+ const files = JSON.parse(readFileSync(pathsFile, "utf8"));
+ if (
+ !Array.isArray(files) ||
+ !files.every((file) => typeof file === "string")
+ ) {
+ throw new Error("Story paths must be a JSON array of strings");
+ }
+ if (files.length === 0) {
+ throw new Error("No stories selected");
+ }
+ const untitled = files.filter(
+ (file) => !hasExplicitTitle(readFileSync(resolve(ROOT, file), "utf8")),
+ );
+ if (untitled.length > 0) {
+ throw new Error(
+ `Stories without an explicit title: ${untitled.join(", ")}`,
+ );
+ }
+ return files.map((file) => `../${file}`);
+ }
+
+ const files = filter
+ ? filter.split(",")
+ : ["frontend/**/*.mdx", ...MAIN_APP_STORY_GLOBS];
+ return files.map((file) => `../${file}`);
+}
+
+module.exports = { MAIN_APP_STORY_GLOBS, getStories, hasExplicitTitle };
diff --git a/eslint.config.mjs b/eslint.config.mjs
index 4f5406b619c..77586954adc 100644
--- a/eslint.config.mjs
+++ b/eslint.config.mjs
@@ -1196,6 +1196,7 @@ const configs = [
".github/scripts/**/*.js",
".github/scripts/**/*.mjs",
".github/scripts/**/*.ts",
+ ".storybook/*.cjs",
],
languageOptions: {
globals: {