The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Provide minimal sane defaults to the Cypress runner (#66398)
Nemanja Glumac637fa143ff10
diff --git a/.github/workflows/e2e-stress-test-flake-fix.yml b/.github/workflows/e2e-stress-test-flake-fix.ymlindex b06dc09c096..fa89eba6412 100644--- a/.github/workflows/e2e-stress-test-flake-fix.yml+++ b/.github/workflows/e2e-stress-test-flake-fix.yml@@ -22,14 +22,10 @@ on:required: falsedefault: "ee"qa_db:- description: "Select the type of Database you want to run your tests with"- type: choice+ description: "Whether to set up and include external QA databases"+ type: booleanrequired: true- default: "none"- options:- - none- - sql- - mongo+ default: falseenable_network_throttling:description: "Enable network throttling simulation (4g slow like in chrome dev tools)"type: boolean@@ -81,8 +77,6 @@ jobs:name: Stress test E2E flake fixenv:DISPLAY: ""- QA_DB_ENABLED: ${{ contains(fromJSON('["sql", "mongo"]'), inputs.qa_db) }}- CYPRESS_QA_DB_MONGO: ${{ inputs.qa_db == 'mongo' }}MB_RUN_MODE: e2eMETASTORE_DEV_SERVER_URL: ${{ secrets.METASTORE_DEV_SERVER_URL }}CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}@@ -109,9 +103,9 @@ jobs:maildev: trueopenldap: truesnowplow: true- postgres: ${{ inputs.qa_db == 'sql' }}- mysql: ${{ inputs.qa_db == 'sql' }}- mongo: ${{ inputs.qa_db == 'mongo' }}+ postgres: ${{ inputs.qa_db }}+ mysql: ${{ inputs.qa_db }}+ mongo: ${{ inputs.qa_db }}- name: Download Metabase ${{ matrix.edition }} uberjaruses: ./.github/actions/e2e-download-uberjar@@ -140,8 +134,11 @@ jobs:uses: ./.github/actions/prepare-cypress- name: Run Metabaserun: node e2e/runner/run_cypress_ci.js start+- name: Generate database snapshots- run: node e2e/runner/run_cypress_ci.js snapshot+ run: |+ node e2e/runner/run_cypress_ci.js snapshot \+ --env grepTags="${{inputs.qa_db && '' || '-@external'}}"# For all options and fine-grained control, take a look at the documentation# https://github.com/cypress-io/cypress/tree/develop/npm/grepdiff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.ymlindex 865f41d23aa..123a19f3e20 100644--- a/.github/workflows/e2e-test.yml+++ b/.github/workflows/e2e-test.yml@@ -37,7 +37,6 @@ jobs:env:MB_EDITION: ${{ inputs.edition }}DISPLAY: ""- QA_DB_ENABLED: truePR_NUMBER: ${{ github.event.pull_request.number || '' }}GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}HASH: ${{ github.event.pull_request.head.sha || github.sha }}-${{ github.run_attempt }}@@ -58,7 +57,6 @@ jobs:MB_SNOWPLOW_URL: "http://localhost:9090" # Snowplow microTZ: US/Pacific # to make node match the instance tzCYPRESS_CI: true- CYPRESS_QA_DB_MONGO: ${{ inputs.name == 'mongo' }}steps:- uses: actions/checkout@v4with:@@ -80,9 +78,9 @@ jobs:openldap: truewebhook: truesnowplow: true- postgres: ${{ inputs.name != 'mongo' }}- mysql: ${{ inputs.name != 'mongo' }}- mongo: ${{ inputs.name == 'mongo' }}+ postgres: true+ mysql: true+ mongo: true- name: Retrieve uberjar artifact for ${{ inputs.edition }}uses: ./.github/actions/fetch-artifactdiff --git a/.github/workflows/embedding-sdk-component-tests.yml b/.github/workflows/embedding-sdk-component-tests.ymlindex 4949ea82289..4173a394a03 100644--- a/.github/workflows/embedding-sdk-component-tests.yml+++ b/.github/workflows/embedding-sdk-component-tests.yml@@ -62,7 +62,10 @@ jobs:- name: Make app db snapshotenv:CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}- run: node e2e/runner/run_cypress_ci.js snapshot+ # Component tests only need the default snapshot+ run: |+ node e2e/runner/run_cypress_ci.js snapshot \+ --spec "e2e/snapshot-creators/default.cy.snap.js"# Has to be run after the app db snapshot is made, as it is not compatible with React 19- name: Change React version to ${{ matrix.react-version }}diff --git a/.github/workflows/embedding-sdk-host-sample-apps-tests.yml b/.github/workflows/embedding-sdk-host-sample-apps-tests.ymlindex 83f96c50aed..62ea23ae99f 100644--- a/.github/workflows/embedding-sdk-host-sample-apps-tests.yml+++ b/.github/workflows/embedding-sdk-host-sample-apps-tests.yml@@ -278,7 +278,9 @@ jobs:run: node e2e/runner/run_cypress_ci.js start- name: Generate database snapshots- run: node e2e/runner/run_cypress_ci.js snapshot+ run: |+ node e2e/runner/run_cypress_ci.js snapshot \+ --spec "e2e/snapshot-creators/default.cy.snap.js"- name: Prepare and launch Host Apprun: npx tsx ./e2e/runner/embedding-sdk/host-apps/start-ci.ts ${{ matrix.test_suite }}diff --git a/cypress.env.json.example b/cypress.env.json.exampleindex 4425d0e42b5..db0e35e4a53 100644--- a/cypress.env.json.example+++ b/cypress.env.json.example@@ -10,5 +10,4 @@"MB_PRO_CLOUD_TOKEN": "get it from the password manager","MB_PRO_SELF_HOSTED_TOKEN": "get it from the password manager","MB_EDITION": "ee",- "QA_DB_ENABLED": "true"}diff --git a/docs/developers-guide/e2e-tests.md b/docs/developers-guide/e2e-tests.mdindex 8e9815624b0..0a4fac78b8d 100644--- a/docs/developers-guide/e2e-tests.md+++ b/docs/developers-guide/e2e-tests.md@@ -122,28 +122,27 @@ One great feature of Cypress is that you can use the Chrome inspector after each`yarn build` and `yarn build-hot` each overwrite an HTML template to reference the correct JavaScript files. If you run `yarn build` before building an Uberjar for Cypress tests, you won’t see changes to your JavaScript reflected even if you then start `yarn build-hot`.-### Running Cypress on M1 machines+### Running Cypress on Apple Silicon++You might run into problems when running Cypress on Apple Silicon processors.-You might run into problems when running Cypress on M1 machine.This is caused by the `@bahmutov/cypress-esbuild-preprocessor` that is using `esbuild` as a dependency. The error might look [like this](https://github.com/evanw/esbuild/issues/1819#issuecomment-1018771557). [The solution](https://github.com/evanw/esbuild/issues/1819#issuecomment-1080720203) is to install NodeJS using one of the Node version managers like [nvm](https://github.com/nvm-sh/nvm) or [n](https://github.com/tj/n).-Another issue you will almost surely face is the inability to connect to our Mongo QA Database. You can solve it by providing the following env:+Another issue you will almost surely face is the inability to connect to our Mongo QA Database. The supported Docker image is incompatible (AMD64). You can solve it by providing the following env:```shellexport EXPERIMENTAL_DOCKER_DESKTOP_FORCE_QEMU=1```-### Running tests that depend on Docker images+Please note that some users experienced Mongo connection timeouts even with this env var set. If that happens, try using OrbStack instead of Docker Desktop.-A subset of our tests depend on the external services that are available through the Docker images. At the time of this writing, those are the three supported external QA databases, Webmail, Snowplow and LDAP servers. The default cypress command will spin up all necessary docker containers for these tests to function properly, but you can toggle them off if you want+### Running tests that depend on Docker images-```sh-START_CONTAINERS=false yarn test-cypress-```+A large portion of our tests depend on the external services that are available through the Docker images. At the time of this writing, those are the three supported external QA databases, Webmail, Snowplow and LDAP servers. See `e2e/test/scenarios/docker-compose.yml` for up to date information. The default cypress command will spin up all necessary Docker containers for the tests to function properly. You can manually set up the e2e environment without them but be aware that you will run into test failures.### Running tests with Snowplow involved-Tests that depend on Snowplow expect a running server. This is enabled by default. You can manually enable them as well by spinning up the snowplow micro docker container and setting the appropriate environment variables:+Tests that depend on Snowplow expect a running server. This is enabled by default. You can manually enable them as well by spinning up the Snowplow micro Docker container and setting the appropriate environment variables:```docker-compose -f ./snowplow/docker-compose.yml up -ddiff --git a/e2e/runner/cypress-runner-backend.js b/e2e/runner/cypress-runner-backend.jsindex fa0da1d1710..06ca1fae1ca 100755--- a/e2e/runner/cypress-runner-backend.js+++ b/e2e/runner/cypress-runner-backend.js@@ -33,6 +33,11 @@ const CypressBackend = {"-Djava.awt.headless=true", // when running on macOS prevent little Java icon from popping up in Dock"-Dmail.smtps.ssl.trust=*", // trust self signed certs for testing"-Duser.timezone=US/Pacific",+ // FIXME @nemanjaglumac 2025-11-28+ // This is a bit convoluted because this particular log4j config outputs all logs to a file (logs/test.log)+ // See: https://github.com/metabase/metabase/pull/21360+ // If we want them in the console, we'd have to unset the log4j config, to let stdio inherit the logs.+ // Override in your .env if you need to see the logs in the console locally.process.env.SHOW_BACKEND_LOGS === "true"? null: `-Dlog4j.configurationFile=file:${__dirname}/../../frontend/test/__runner__/log4j2.xml`,@@ -50,6 +55,8 @@ const CypressBackend = {MB_CONFIG_FILE_PATH: "__cypress__", // ignore config.ymlMB_HTTP_CHANNEL_HOST_STRATEGY: "allow-all", // we use a local webhook service for testingMB_IS_CYPRESS: "true", // custom flag so we can detect we're running in Cypress, used in tests.+ MB_SNOWPLOW_AVAILABLE: true,+ MB_SNOWPLOW_URL: "http://localhost:9090",};this.server.process = spawn("java", [...javaFlags, "-jar", jarPath], {diff --git a/e2e/runner/run_cypress_ci.js b/e2e/runner/run_cypress_ci.jsindex dcafaac3a35..c3adb7572db 100755--- a/e2e/runner/run_cypress_ci.js+++ b/e2e/runner/run_cypress_ci.js@@ -46,7 +46,10 @@ if (command === "start") {}if (command === "snapshot") {- runTests({ configFile: "e2e/support/cypress-snapshots.config.js" });+ runTests(+ { configFile: "e2e/support/cypress-snapshots.config.js" },+ cliArguments,+ );}// Metabase component or e2e testsdiff --git a/e2e/runner/run_cypress_local.ts b/e2e/runner/run_cypress_local.tsindex 64e1e00762b..33ca35c28fa 100644--- a/e2e/runner/run_cypress_local.ts+++ b/e2e/runner/run_cypress_local.ts@@ -1,5 +1,3 @@-import fs from "fs";-import os from "os";import path from "path";import { BACKEND_PORT } from "./constants/backend-port";@@ -14,103 +12,72 @@ import {unBooleanify,} from "./cypress-runner-utils";-let tempSampleDBDir: string | null = null;-// if you want to change these, set them as environment variables in your shell-const userOptions = {+const options = {CYPRESS_TESTING_TYPE: "e2e", // e2e | componentMB_EDITION: "ee", // ee | oss- START_CONTAINERS: true,- STOP_CONTAINERS: false,+ BUILD_BACKEND: true,BACKEND_PORT: BACKEND_PORT, // override with MB_JETTY_PORT in your envOPEN_UI: true,- SHOW_BACKEND_LOGS: false,GENERATE_SNAPSHOTS: true,- QUIET: false,- TZ: "UTC",...booleanify(process.env),};-const derivedOptions = {- QA_DB_ENABLED: userOptions.START_CONTAINERS,- BUILD_JAR: userOptions.BACKEND_PORT === 4000,- START_BACKEND: userOptions.BACKEND_PORT === 4000,- MB_SNOWPLOW_AVAILABLE: true,- MB_SNOWPLOW_URL: "http://localhost:9090",-};--const options = {- ...derivedOptions,- ...userOptions,-};-process.env = unBooleanify(options);const missingTokens = [- "MB_ALL_FEATURES_TOKEN",- "MB_STARTER_CLOUD_TOKEN",- "MB_PRO_CLOUD_TOKEN",- "MB_PRO_SELF_HOSTED_TOKEN",+ "CYPRESS_MB_ALL_FEATURES_TOKEN",+ "CYPRESS_MB_STARTER_CLOUD_TOKEN",+ "CYPRESS_MB_PRO_CLOUD_TOKEN",+ "CYPRESS_MB_PRO_SELF_HOSTED_TOKEN",].filter((token) => !process.env[token]);if (options.MB_EDITION === "ee" && missingTokens.length > 0) {printBold(`⚠️ Missing tokens: ${missingTokens.join(", ")}. Either set them or run with MB_EDITION=oss`,);- process.exit(FAILURE_EXIT_CODE);}printBold(`Running Cypress with options:- CYPRESS_TESTING_TYPE : ${options.CYPRESS_TESTING_TYPE}- MB_EDITION : ${options.MB_EDITION}- - START_CONTAINERS : ${options.START_CONTAINERS}- - STOP_CONTAINERS : ${options.STOP_CONTAINERS}- - BUILD_JAR : ${options.BUILD_JAR}- - GENERATE_SNAPSHOTS : ${options.GENERATE_SNAPSHOTS}- BACKEND_PORT : ${options.BACKEND_PORT}- - START_BACKEND : ${options.START_BACKEND}+ - BUILD_BACKEND : ${options.BUILD_BACKEND}+ - GENERATE_SNAPSHOTS : ${options.GENERATE_SNAPSHOTS}- OPEN_UI : ${options.OPEN_UI}- - SHOW_BACKEND_LOGS : ${options.SHOW_BACKEND_LOGS}- - TZ : ${options.TZ}`);const init = async () => {const cliArguments = process.argv.slice(2);const userOverrides = await parseArguments(cliArguments);- if (options.START_CONTAINERS) {- printBold("⏳ Starting containers");- shell("docker compose -f ./e2e/test/scenarios/docker-compose.yml up -d");- }+ printBold("⏳ Starting containers");+ shell("docker compose -f ./e2e/test/scenarios/docker-compose.yml up -d");- if (options.BUILD_JAR) {+ if (options.BUILD_BACKEND) {printBold("⏳ Building backend");shell("./bin/build-backend-for-test");- if (options.START_BACKEND) {- const isBackendRunning = shell(- `lsof -ti:${options.BACKEND_PORT} || echo ""`,- { quiet: true },- );- if (isBackendRunning) {- printBold(- "⚠️ Your backend is already running, you may want to kill pid " +- isBackendRunning,- );- process.exit(FAILURE_EXIT_CODE);- }-- // Use a temporary copy of the sample db so it won't use and lock the db used for local development- tempSampleDBDir = path.join(- os.tmpdir(),- `metabase-sample-db-e2e-${process.pid}`,+ const isBackendRunning = shell(+ `lsof -ti:${options.BACKEND_PORT} || echo ""`,+ { quiet: true },+ );+ if (isBackendRunning) {+ printBold(+ "⚠️ Your backend is already running, you may want to kill pid " ++ isBackendRunning,);- fs.mkdirSync(tempSampleDBDir, { recursive: true });- process.env.MB_INTERNAL_DO_NOT_USE_SAMPLE_DB_DIR = tempSampleDBDir;-- printBold("⏳ Starting backend");- await CypressBackend.start("target/uberjar/metabase-backend.jar");+ process.exit(FAILURE_EXIT_CODE);}++ // Use a temporary copy of the sample db so it won't use and lock the db used for local development+ process.env.MB_INTERNAL_DO_NOT_USE_SAMPLE_DB_DIR = path.resolve(+ __dirname,+ "../../e2e/tmp", // already exists and is .gitignored+ );++ printBold("⏳ Starting backend");+ await CypressBackend.start("target/uberjar/metabase-backend.jar");} else {printBold(`Not building a jar, expecting metabase to be running on port ${options.BACKEND_PORT}. Make sure your metabase instance is running with an h2 app db and the following environment variables:@@ -126,7 +93,12 @@ const init = async () => {printBold("⏳ Generating app db snapshots");process.env.OPEN_UI = "false";- await runCypress({ configFile: "e2e/support/cypress-snapshots.config.js" });+ await runCypress({+ configFile: "e2e/support/cypress-snapshots.config.js",+ ...(options.CYPRESS_TESTING_TYPE === "component" && {+ env: { grepTags: "-@external" }, // component tests do not need QA DB snapshots for now+ }),+ });process.env.OPEN_UI = `${options.OPEN_UI}`;} else {printBold("Skipping snapshot generation, beware of stale snapshot caches");@@ -157,29 +129,14 @@ const init = async () => {};const cleanup = async (exitCode: string | number = SUCCESS_EXIT_CODE) => {- if (options.BUILD_JAR) {+ if (options.BUILD_BACKEND) {printBold("⏳ Cleaning up...");await CypressBackend.stop();}- // Add cleanup for the temporary sample database directory- if (tempSampleDBDir) {- try {- fs.rmSync(tempSampleDBDir, { recursive: true, force: true });- printBold(- `🗑️ Cleaned up temporary sample database directory: ${tempSampleDBDir}`,- );- } catch (e) {- console.error(- `Error cleaning up temporary sample database directory: ${e}`,- );- }- }-- if (options.STOP_CONTAINERS) {- printBold("⏳ Stopping containers");- shell("docker compose -f ./e2e/test/scenarios/docker-compose.yml down");- }+ printBold(+ "🧹 Containers are running in background. If you wish to stop them, run:\n`docker compose -f ./e2e/test/scenarios/docker-compose.yml down`",+ );typeof exitCode === "number"? process.exit(exitCode)diff --git a/e2e/snapshot-creators/qa-db.cy.snap.js b/e2e/snapshot-creators/qa-db.cy.snap.jsindex 25c6f75f7eb..ec5a7f2c1dd 100644--- a/e2e/snapshot-creators/qa-db.cy.snap.js+++ b/e2e/snapshot-creators/qa-db.cy.snap.js@@ -9,29 +9,29 @@ import {describe("qa databases snapshots", { tags: "@external" }, () => {it("creates snapshots for supported qa databases", () => {- if (Cypress.env("QA_DB_MONGO") === true) {- restoreAndAuthenticate();- addMongoDatabase();- snapshot("mongo-5");- } else {- // Postgres- restoreAndAuthenticate();+ // Mongo+ restoreAndAuthenticate();- addPostgresDatabase();- snapshot("postgres-12");+ addMongoDatabase();+ snapshot("mongo-5");- convertToWritable("postgres");- snapshot("postgres-writable");+ // Postgres+ restoreAndAuthenticate();- // MySQL- restoreAndAuthenticate();+ addPostgresDatabase();+ snapshot("postgres-12");- addMySQLDatabase({});- snapshot("mysql-8");+ convertToWritable("postgres");+ snapshot("postgres-writable");- convertToWritable("mysql");- snapshot("mysql-writable");- }+ // MySQL+ restoreAndAuthenticate();++ addMySQLDatabase({});+ snapshot("mysql-8");++ convertToWritable("mysql");+ snapshot("mysql-writable");restore("blank");});diff --git a/e2e/support/cypress-snapshots.config.js b/e2e/support/cypress-snapshots.config.jsindex aee6ee38f67..cb5166a01ac 100644--- a/e2e/support/cypress-snapshots.config.js+++ b/e2e/support/cypress-snapshots.config.js@@ -2,15 +2,10 @@ const { defineConfig } = require("cypress");const { defaultConfig } = require("./config");-const isQaDatabase = process.env["QA_DB_ENABLED"] === "true";-module.exports = defineConfig({e2e: {...defaultConfig,specPattern: "e2e/snapshot-creators/**/*.cy.snap.js",- excludeSpecPattern: !isQaDatabase- ? "e2e/snapshot-creators/qa-db.cy.snap.js"- : undefined,video: false,},});diff --git a/e2e/support/helpers/e2e-setup-helpers.js b/e2e/support/helpers/e2e-setup-helpers.jsindex c0bca396e20..25a7529aa88 100644--- a/e2e/support/helpers/e2e-setup-helpers.js+++ b/e2e/support/helpers/e2e-setup-helpers.js@@ -19,8 +19,6 @@ export function snapshot(name) {* } name*/export function restore(name = "default") {- cy.skipOn(name.includes("mongo") && Cypress.env("QA_DB_MONGO") !== true);-cy.log("Restore Data Set");// automatically reset the data db if this is a test that uses a writable db