The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Generate default E2E snapshots once per run (#84074)
Chris Vandoodewaard0bc186ade87c
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.ymlindex b0702eefaff..f857247e8b3 100644--- a/.github/workflows/e2e-test.yml+++ b/.github/workflows/e2e-test.yml@@ -19,6 +19,10 @@ on:description: "ID of this run's test-plan.json artifact, '' to use the default spec pattern."type: stringdefault: ""+ snapshots-artifact:+ description: "Name of this run's default snapshots artifact, '' to generate all snapshots in this job."+ type: string+ default: ""tags:required: falsetype: string@@ -185,7 +189,17 @@ jobs:# steps below. DEV-1999.- uses: ./.github/actions/resolve-job-id+ - name: Download database snapshots+ id: snapshots+ if: ${{ inputs.snapshots-artifact != '' && inputs.edition == 'ee' }}+ uses: actions/download-artifact@v8+ continue-on-error: true+ with:+ name: ${{ inputs.snapshots-artifact }}+ path: e2e+- name: Generate database snapshots+ if: ${{ steps.snapshots.outcome != 'success' }}env:CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}run: node e2e/runner/run_cypress_ci.js snapshotdiff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.ymlindex 0ee8573b913..2183c576034 100644--- a/.github/workflows/e2e-tests.yml+++ b/.github/workflows/e2e-tests.yml@@ -27,14 +27,100 @@ jobs:chunks: 50test-plan-artifact-id: ${{ inputs.test-plan-artifact-id }}+ e2e-snapshots:+ name: Generate E2E snapshots+ needs: e2e-matrix-builder+ if: ${{ needs.e2e-matrix-builder.outputs.hasTests == 'true' }}+ continue-on-error: true+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}+ timeout-minutes: 15+ env:+ MB_EDITION: ee+ CYPRESS_VIDEO: "false"+ CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}+ steps:+ - uses: actions/checkout@v7+ with:+ submodules: true+ token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}++ - name: Prepare Docker containers+ uses: ./.github/actions/e2e-prepare-containers+ with:+ username: ${{ secrets.DOCKERHUB_USERNAME }}+ password: ${{ secrets.DOCKERHUB_TOKEN }}+ maildev: false+ openldap: false+ mocksaml: false+ postgres: true+ mysql: true+ mongo: true++ - name: Retrieve uberjar artifact for ee+ uses: ./.github/actions/fetch-artifact+ with:+ commit: ${{ github.event.pull_request.head.sha || github.sha }}+ edition: ee+ extract-path: "target/uberjar"+ output-name: "metabase.jar"++ - name: Prepare front-end environment+ uses: ./.github/actions/prepare-frontend++ - name: Prepare JDK+ uses: actions/[email protected]+ with:+ java-version: "25"+ distribution: "temurin"++ - name: Prepare Cypress environment+ id: cypress-prep+ uses: ./.github/actions/prepare-cypress++ - name: Download pre-built CLJS+ id: prebuilt-cljs+ uses: actions/download-artifact@v8+ continue-on-error: true+ with:+ name: e2e-cljs-${{ github.event.pull_request.head.sha || github.sha }}+ path: target/cljs_dev++ - name: Prepare back-end environment+ if: steps.prebuilt-cljs.outcome != 'success'+ uses: ./.github/actions/prepare-backend++ - name: Compile CLJS+ if: steps.prebuilt-cljs.outcome != 'success'+ run: bun run build-pure:cljs++ - name: Run Metabase+ run: node e2e/runner/run_cypress_ci.js start++ - name: Generate database snapshots+ env:+ CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}+ run: node e2e/runner/run_cypress_ci.js snapshot++ - name: Upload snapshots+ uses: actions/upload-artifact@v7+ with:+ name: e2e-snapshots-ee-${{ github.event.pull_request.head.sha || github.sha }}+ path: |+ e2e/snapshots/+ e2e/support/cypress_sample_database.json+ e2e/support/cypress_sample_instance_data.json+ if-no-files-found: error+ overwrite: true+e2e-tests:name: ${{ matrix.name }}-${{ matrix.edition }}uses: ./.github/workflows/e2e-test.ymlneeds:- e2e-matrix-builder- # No status function, so a skipped or failed matrix builder keeps this out: the strategy below- # cannot parse a matrix it never produced. The builder itself only runs when the gate says so.- if: ${{ needs.e2e-matrix-builder.outputs.hasTests == 'true' }}+ - e2e-snapshots+ # Run even if e2e-snapshots fails: each shard then generates its own snapshots. The matrix builder+ # must succeed, because the strategy below cannot parse a matrix it never produced.+ if: ${{ !cancelled() && needs.e2e-matrix-builder.result == 'success' && needs.e2e-matrix-builder.outputs.hasTests == 'true' }}strategy:fail-fast: falsematrix: ${{ fromJson(needs.e2e-matrix-builder.outputs.matrix) }}@@ -44,6 +130,7 @@ jobs:edition: ${{ matrix.edition }}specs: ${{ matrix.specs }}test-plan-artifact-id: ${{ needs.e2e-matrix-builder.outputs.test-plan-artifact-id }}+ snapshots-artifact: e2e-snapshots-ee-${{ github.event.pull_request.head.sha || github.sha }}total_chunks: ${{ fromJson(needs.e2e-matrix-builder.outputs.regularChunks) }}split_index: ${{ strategy.job-index }}tags: ${{ matrix.tags }}diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.ymlindex 03feec6a760..e44db4bd860 100644--- a/.github/workflows/run-tests.yml+++ b/.github/workflows/run-tests.yml@@ -163,7 +163,7 @@ jobs:test-plan-artifact-id: ${{ needs.create-test-plan.outputs.artifact-id }}e2e-tests:- needs: [gate, uberjar, create-test-plan]+ needs: [gate, uberjar, e2e-cljs, create-test-plan]# Skip e2e tests when the uberjar build failed or was cancelled — the# downstream tests can't fetch a missing artifact, and we don't want to# mask the uberjar failure with a wave of misleading e2e errors. We stilldiff --git a/e2e/support/db_tasks.js b/e2e/support/db_tasks.jsindex 3efcd6dd84e..67906ba06dc 100644--- a/e2e/support/db_tasks.js+++ b/e2e/support/db_tasks.js@@ -5,10 +5,13 @@ import { Roles } from "./test_roles";import * as testTables from "./test_tables";const dbClients = {};+const existingWritableDbs = new Set();function getDbClient(connectionConfig) {const connectionId =- connectionConfig.client + connectionConfig.connection.database;+ connectionConfig.client ++ connectionConfig.connection.database ++ connectionConfig.connection.user;// we only need to create one client per db per runif (!dbClients[connectionId]) {@@ -19,6 +22,11 @@ function getDbClient(connectionConfig) {}export async function connectAndQueryDB({ connectionConfig, query }) {+ const writableDbType = getWritableDbType(connectionConfig);+ if (writableDbType) {+ await createWritableDb({ type: writableDbType });+ }+const dbClient = getDbClient(connectionConfig);const result = await dbClient.raw(query);@@ -33,12 +41,24 @@ export async function connectAndQueryDB({ connectionConfig, query }) {}}+function getWritableDbType(connectionConfig) {+ return Object.keys(WRITABLE_DB_CONFIG).find((type) => {+ const { client, connection } = WRITABLE_DB_CONFIG[type];+ return (+ client === connectionConfig.client &&+ connection.database === connectionConfig.connection.database+ );+ });+}+/*** For postgres, drops all schemas except for public, and drops all tables from the public schema.** For mysql, drops all user created tables from the writable_db database.*/export async function resetWritableDb({ type = "postgres" }) {+ await createWritableDb({ type });+const dbClient = getDbClient(WRITABLE_DB_CONFIG[type]);if (type === "postgres") {@@ -67,7 +87,7 @@ export async function resetWritableDb({ type = "postgres" }) {}for (const table of tables) {- await dbClient.raw(`DROP TABLE public.${table.name};`);+ await dbClient.raw(`DROP TABLE IF EXISTS public.${table.name} CASCADE;`);}} else if (type === "mysql") {const { rows: tables } = await dbClient.raw(@@ -85,7 +105,66 @@ export async function resetWritableDb({ type = "postgres" }) {return null;}+function getAdminConnectionConfig(type) {+ return {+ postgres: QA_DB_CONFIG.postgres,+ mysql: {+ ...QA_DB_CONFIG.mysql,+ connection: { ...QA_DB_CONFIG.mysql.connection, user: "root" },+ },+ }[type];+}++async function writableDbExists(type) {+ if (existingWritableDbs.has(type)) {+ return true;+ }++ const dbName = WRITABLE_DB_CONFIG[type].connection.database;++ const dbCheckQuery = {+ postgres: `SELECT FROM pg_database WHERE datname = '${dbName}';`,+ mysql: `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='${dbName}'`,+ }[type];++ const { rows } = await connectAndQueryDB({+ connectionConfig: getAdminConnectionConfig(type),+ query: dbCheckQuery,+ });++ const exists = rows.length > 0;+ if (exists) {+ existingWritableDbs.add(type);+ }++ return exists;+}++async function assertWritableDbExists(type) {+ if (!(await writableDbExists(type))) {+ const dbName = WRITABLE_DB_CONFIG[type].connection.database;+ throw new Error(+ `The ${type} "${dbName}" database does not exist. Restore the "${type}-writable" snapshot or call setupWritableDB("${type}") first.`,+ );+ }+}++export async function createWritableDb({ type = "postgres" }) {+ if (!(await writableDbExists(type))) {+ const dbName = WRITABLE_DB_CONFIG[type].connection.database;+ await connectAndQueryDB({+ connectionConfig: getAdminConnectionConfig(type),+ query: `CREATE DATABASE ${dbName};`,+ });+ existingWritableDbs.add(type);+ }++ return null;+}+export async function resetTable({ type = "postgres", table = "testTable1" }) {+ await assertWritableDbExists(type);+const dbClient = getDbClient(WRITABLE_DB_CONFIG[type]);return testTables?.[table]?.(dbClient);}@@ -94,6 +173,10 @@ export async function createTestRoles({type = "postgres",isWritable = false,}) {+ if (isWritable) {+ await assertWritableDbExists(type);+ }+const config = isWritable ? WRITABLE_DB_CONFIG : QA_DB_CONFIG;const dbClient = getDbClient(config[type]);diff --git a/e2e/support/helpers/e2e-qa-databases-helpers.js b/e2e/support/helpers/e2e-qa-databases-helpers.jsindex c0bba2b331c..2e453e821ee 100644--- a/e2e/support/helpers/e2e-qa-databases-helpers.js+++ b/e2e/support/helpers/e2e-qa-databases-helpers.js@@ -206,44 +206,7 @@ function recursiveCheckFields(id, i = 0) {}export const setupWritableDB = (type = "postgres") => {- const connectionConfig = {- postgres: {- client: "pg",- connection: {- ...QA_DB_CREDENTIALS,- port: QA_POSTGRES_PORT,- },- },- mysql: {- client: "mysql2",- connection: {- ...QA_DB_CREDENTIALS,- user: "root",- port: QA_MYSQL_PORT,- },- },- };-- const dbName = WRITABLE_DB_CONFIG[type].connection.database;-- const dbCheckQuery = {- postgres: `SELECT FROM pg_database WHERE datname = '${dbName}';`,- mysql: `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='${dbName}'`,- };-- // we need to initially connect to the db we know exists to create the writable_db- cy.task("connectAndQueryDB", {- connectionConfig: connectionConfig[type],- query: dbCheckQuery[type],- }).then((results) => {- if (!results.rows.length) {- cy.log(`**-- Adding ${type} DB for actions --**`);- cy.task("connectAndQueryDB", {- connectionConfig: connectionConfig[type],- query: `CREATE DATABASE ${dbName};`,- });- }- });+ cy.task("createWritableDb", { type });};export function queryQADB(query, type = "postgres") {diff --git a/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.ts b/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.tsindex 2ec1f1a4ac7..277b92660b8 100644--- a/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.ts+++ b/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.ts@@ -103,6 +103,10 @@ function mapDatabricksValues(parsedValues: RegexFields) {]),);+ if (fieldsMap.get("details.additional-options")) {+ fieldsMap.set("details.advanced-options", true);+ }+return fieldsMap;}@@ -127,9 +131,14 @@ function mapMysqlValues(parsedValues: RegexFields) {["details.ssl", parsedValues.params?.ssl],["details.additional-options",- objectToString(parsedValues.params, ["ssl"]),+ objectToString(parsedValues.params, ["ssl"], "&"),],]);++ if (fieldsMap.get("details.additional-options")) {+ fieldsMap.set("details.advanced-options", true);+ }+return fieldsMap;}@@ -156,10 +165,14 @@ function mapPostgresValues(parsedValues: RegexFields) {["details.ssl", parsedValues.params?.ssl],["details.additional-options",- objectToString(parsedValues.params, ["ssl"]),+ objectToString(parsedValues.params, ["ssl"], "&"),],]);+ if (fieldsMap.get("details.additional-options")) {+ fieldsMap.set("details.advanced-options", true);+ }+return fieldsMap;}@@ -389,14 +402,16 @@ export function mapFieldsToNestedObject(}/**- * Converts an object to a semicolon-separated string.+ * Converts an object to a string of key=value pairs.* @param obj - The object to convert.* @param filterProperties - The properties to filter out.- * @returns The semicolon-separated string.+ * @param separator - The string between pairs.+ * @returns The joined string.*/function objectToString(obj: Record<string, string | undefined> | undefined,filterProperties: Array<string> = [],+ separator = ";",) {if (!obj) {return "";@@ -405,5 +420,5 @@ function objectToString(return Object.entries(obj).filter(([key]) => !filterProperties.includes(key)).map(([key, value]) => `${key}=${value}`)- .join(";");+ .join(separator);}diff --git a/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.unit.spec.ts b/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.unit.spec.tsindex 9737523223f..29edf2b0c3c 100644--- a/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.unit.spec.ts+++ b/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.unit.spec.ts@@ -1,15 +1,78 @@+import { getSubmitValues } from "metabase/databases/utils/schema";import { checkNotNull } from "metabase/utils/types";+import type { EngineKey } from "metabase-types/api";+import {+ createMockDatabaseData,+ createMockEngine,+} from "metabase-types/api/mocks";import { mapDatabaseValues } from "./database-field-mapper";import { parseConnectionUriRegex } from "./parse-connection-regex";-function mapConnectionString(connectionString: string, engineKey: "redshift") {+function mapConnectionString(+ connectionString: string,+ engineKey: EngineKey,+): Map<string, string | boolean | undefined> {const parsedValues = checkNotNull(parseConnectionUriRegex(connectionString, engineKey),);return mapDatabaseValues(parsedValues, engineKey);}+describe.each<{+ engineKey: EngineKey;+ connectionString: string;+ additionalOptions: string;+}>([+ {+ engineKey: "mysql",+ connectionString:+ "jdbc:mysql://metabase:password@localhost:3304/sample?allowPublicKeyRetrieval=true&useSSL=false",+ additionalOptions: "allowPublicKeyRetrieval=true&useSSL=false",+ },+ {+ engineKey: "postgres",+ connectionString:+ "jdbc:postgresql://metabase:password@localhost:5404/sample?prepareThreshold=0&connectTimeout=10",+ additionalOptions: "prepareThreshold=0&connectTimeout=10",+ },+ {+ engineKey: "databricks",+ connectionString:+ "jdbc:databricks://example.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/abc;IgnoreTransactions=0",+ additionalOptions: "IgnoreTransactions=0",+ },+])(+ "mapDatabaseValues - $engineKey",+ ({ engineKey, connectionString, additionalOptions }) => {+ it("should preserve additional options in submitted connection details", () => {+ const fieldsMap = mapConnectionString(connectionString, engineKey);+ const engine = createMockEngine({+ "details-fields": [+ {+ name: "additional-options",+ "visible-if": { "advanced-options": true },+ },+ ],+ });+ const submitted = getSubmitValues(+ engine,+ createMockDatabaseData({+ name: "sample",+ engine: engineKey,+ details: {+ "advanced-options": fieldsMap.get("details.advanced-options"),+ "additional-options": fieldsMap.get("details.additional-options"),+ },+ }),+ true,+ );++ expect(submitted.details?.["additional-options"]).toBe(additionalOptions);+ });+ },+);+describe("mapDatabaseValues - redshift", () => {it("should map UID and PWD to dedicated fields and keep unmapped params in additional options", () => {const fieldsMap = mapConnectionString(