Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Generate default E2E snapshots once per run (#84074)

Chris Vandoodewaard0bc186ade87c
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index b0702eefaff..f857247e8b3 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -19,6 +19,10 @@ on:
description: "ID of this run's test-plan.json artifact, '' to use the default spec pattern."
type: string
default: ""
+ snapshots-artifact:
+ description: "Name of this run's default snapshots artifact, '' to generate all snapshots in this job."
+ type: string
+ default: ""
tags:
required: false
type: string
@@ -185,7 +189,17 @@ jobs:
# steps below. DEV-1999.
- uses: ./.github/actions/resolve-job-id
+ - name: Download database snapshots
+ id: snapshots
+ if: ${{ inputs.snapshots-artifact != '' && inputs.edition == 'ee' }}
+ uses: actions/download-artifact@v8
+ continue-on-error: true
+ with:
+ name: ${{ inputs.snapshots-artifact }}
+ path: e2e
+
- name: Generate database snapshots
+ if: ${{ steps.snapshots.outcome != 'success' }}
env:
CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}
run: node e2e/runner/run_cypress_ci.js snapshot
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
index 0ee8573b913..2183c576034 100644
--- a/.github/workflows/e2e-tests.yml
+++ b/.github/workflows/e2e-tests.yml
@@ -27,14 +27,100 @@ jobs:
chunks: 50
test-plan-artifact-id: ${{ inputs.test-plan-artifact-id }}
+ e2e-snapshots:
+ name: Generate E2E snapshots
+ needs: e2e-matrix-builder
+ if: ${{ needs.e2e-matrix-builder.outputs.hasTests == 'true' }}
+ continue-on-error: true
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 15
+ env:
+ MB_EDITION: ee
+ CYPRESS_VIDEO: "false"
+ CYPRESS_MB_ALL_FEATURES_TOKEN: ${{ secrets.STAGING_MB_ALL_FEATURES_TOKEN }}
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ submodules: true
+ token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
+
+ - name: Prepare Docker containers
+ uses: ./.github/actions/e2e-prepare-containers
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+ maildev: false
+ openldap: false
+ mocksaml: false
+ postgres: true
+ mysql: true
+ mongo: true
+
+ - name: Retrieve uberjar artifact for ee
+ uses: ./.github/actions/fetch-artifact
+ with:
+ commit: ${{ github.event.pull_request.head.sha || github.sha }}
+ edition: ee
+ extract-path: "target/uberjar"
+ output-name: "metabase.jar"
+
+ - name: Prepare front-end environment
+ uses: ./.github/actions/prepare-frontend
+
+ - name: Prepare JDK
+ uses: actions/[email protected]
+ with:
+ java-version: "25"
+ distribution: "temurin"
+
+ - name: Prepare Cypress environment
+ id: cypress-prep
+ uses: ./.github/actions/prepare-cypress
+
+ - name: Download pre-built CLJS
+ id: prebuilt-cljs
+ uses: actions/download-artifact@v8
+ continue-on-error: true
+ with:
+ name: e2e-cljs-${{ github.event.pull_request.head.sha || github.sha }}
+ path: target/cljs_dev
+
+ - name: Prepare back-end environment
+ if: steps.prebuilt-cljs.outcome != 'success'
+ uses: ./.github/actions/prepare-backend
+
+ - name: Compile CLJS
+ if: steps.prebuilt-cljs.outcome != 'success'
+ run: bun run build-pure:cljs
+
+ - name: Run Metabase
+ run: node e2e/runner/run_cypress_ci.js start
+
+ - name: Generate database snapshots
+ env:
+ CYPRESS_BROWSER: ${{ steps.cypress-prep.outputs.chrome-path }}
+ run: node e2e/runner/run_cypress_ci.js snapshot
+
+ - name: Upload snapshots
+ uses: actions/upload-artifact@v7
+ with:
+ name: e2e-snapshots-ee-${{ github.event.pull_request.head.sha || github.sha }}
+ path: |
+ e2e/snapshots/
+ e2e/support/cypress_sample_database.json
+ e2e/support/cypress_sample_instance_data.json
+ if-no-files-found: error
+ overwrite: true
+
e2e-tests:
name: ${{ matrix.name }}-${{ matrix.edition }}
uses: ./.github/workflows/e2e-test.yml
needs:
- e2e-matrix-builder
- # No status function, so a skipped or failed matrix builder keeps this out: the strategy below
- # cannot parse a matrix it never produced. The builder itself only runs when the gate says so.
- if: ${{ needs.e2e-matrix-builder.outputs.hasTests == 'true' }}
+ - e2e-snapshots
+ # Run even if e2e-snapshots fails: each shard then generates its own snapshots. The matrix builder
+ # must succeed, because the strategy below cannot parse a matrix it never produced.
+ if: ${{ !cancelled() && needs.e2e-matrix-builder.result == 'success' && needs.e2e-matrix-builder.outputs.hasTests == 'true' }}
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.e2e-matrix-builder.outputs.matrix) }}
@@ -44,6 +130,7 @@ jobs:
edition: ${{ matrix.edition }}
specs: ${{ matrix.specs }}
test-plan-artifact-id: ${{ needs.e2e-matrix-builder.outputs.test-plan-artifact-id }}
+ snapshots-artifact: e2e-snapshots-ee-${{ github.event.pull_request.head.sha || github.sha }}
total_chunks: ${{ fromJson(needs.e2e-matrix-builder.outputs.regularChunks) }}
split_index: ${{ strategy.job-index }}
tags: ${{ matrix.tags }}
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 03feec6a760..e44db4bd860 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -163,7 +163,7 @@ jobs:
test-plan-artifact-id: ${{ needs.create-test-plan.outputs.artifact-id }}
e2e-tests:
- needs: [gate, uberjar, create-test-plan]
+ needs: [gate, uberjar, e2e-cljs, create-test-plan]
# Skip e2e tests when the uberjar build failed or was cancelled — the
# downstream tests can't fetch a missing artifact, and we don't want to
# mask the uberjar failure with a wave of misleading e2e errors. We still
diff --git a/e2e/support/db_tasks.js b/e2e/support/db_tasks.js
index 3efcd6dd84e..67906ba06dc 100644
--- a/e2e/support/db_tasks.js
+++ b/e2e/support/db_tasks.js
@@ -5,10 +5,13 @@ import { Roles } from "./test_roles";
import * as testTables from "./test_tables";
const dbClients = {};
+const existingWritableDbs = new Set();
function getDbClient(connectionConfig) {
const connectionId =
- connectionConfig.client + connectionConfig.connection.database;
+ connectionConfig.client +
+ connectionConfig.connection.database +
+ connectionConfig.connection.user;
// we only need to create one client per db per run
if (!dbClients[connectionId]) {
@@ -19,6 +22,11 @@ function getDbClient(connectionConfig) {
}
export async function connectAndQueryDB({ connectionConfig, query }) {
+ const writableDbType = getWritableDbType(connectionConfig);
+ if (writableDbType) {
+ await createWritableDb({ type: writableDbType });
+ }
+
const dbClient = getDbClient(connectionConfig);
const result = await dbClient.raw(query);
@@ -33,12 +41,24 @@ export async function connectAndQueryDB({ connectionConfig, query }) {
}
}
+function getWritableDbType(connectionConfig) {
+ return Object.keys(WRITABLE_DB_CONFIG).find((type) => {
+ const { client, connection } = WRITABLE_DB_CONFIG[type];
+ return (
+ client === connectionConfig.client &&
+ connection.database === connectionConfig.connection.database
+ );
+ });
+}
+
/**
* For postgres, drops all schemas except for public, and drops all tables from the public schema.
*
* For mysql, drops all user created tables from the writable_db database.
*/
export async function resetWritableDb({ type = "postgres" }) {
+ await createWritableDb({ type });
+
const dbClient = getDbClient(WRITABLE_DB_CONFIG[type]);
if (type === "postgres") {
@@ -67,7 +87,7 @@ export async function resetWritableDb({ type = "postgres" }) {
}
for (const table of tables) {
- await dbClient.raw(`DROP TABLE public.${table.name};`);
+ await dbClient.raw(`DROP TABLE IF EXISTS public.${table.name} CASCADE;`);
}
} else if (type === "mysql") {
const { rows: tables } = await dbClient.raw(
@@ -85,7 +105,66 @@ export async function resetWritableDb({ type = "postgres" }) {
return null;
}
+function getAdminConnectionConfig(type) {
+ return {
+ postgres: QA_DB_CONFIG.postgres,
+ mysql: {
+ ...QA_DB_CONFIG.mysql,
+ connection: { ...QA_DB_CONFIG.mysql.connection, user: "root" },
+ },
+ }[type];
+}
+
+async function writableDbExists(type) {
+ if (existingWritableDbs.has(type)) {
+ return true;
+ }
+
+ const dbName = WRITABLE_DB_CONFIG[type].connection.database;
+
+ const dbCheckQuery = {
+ postgres: `SELECT FROM pg_database WHERE datname = '${dbName}';`,
+ mysql: `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='${dbName}'`,
+ }[type];
+
+ const { rows } = await connectAndQueryDB({
+ connectionConfig: getAdminConnectionConfig(type),
+ query: dbCheckQuery,
+ });
+
+ const exists = rows.length > 0;
+ if (exists) {
+ existingWritableDbs.add(type);
+ }
+
+ return exists;
+}
+
+async function assertWritableDbExists(type) {
+ if (!(await writableDbExists(type))) {
+ const dbName = WRITABLE_DB_CONFIG[type].connection.database;
+ throw new Error(
+ `The ${type} "${dbName}" database does not exist. Restore the "${type}-writable" snapshot or call setupWritableDB("${type}") first.`,
+ );
+ }
+}
+
+export async function createWritableDb({ type = "postgres" }) {
+ if (!(await writableDbExists(type))) {
+ const dbName = WRITABLE_DB_CONFIG[type].connection.database;
+ await connectAndQueryDB({
+ connectionConfig: getAdminConnectionConfig(type),
+ query: `CREATE DATABASE ${dbName};`,
+ });
+ existingWritableDbs.add(type);
+ }
+
+ return null;
+}
+
export async function resetTable({ type = "postgres", table = "testTable1" }) {
+ await assertWritableDbExists(type);
+
const dbClient = getDbClient(WRITABLE_DB_CONFIG[type]);
return testTables?.[table]?.(dbClient);
}
@@ -94,6 +173,10 @@ export async function createTestRoles({
type = "postgres",
isWritable = false,
}) {
+ if (isWritable) {
+ await assertWritableDbExists(type);
+ }
+
const config = isWritable ? WRITABLE_DB_CONFIG : QA_DB_CONFIG;
const dbClient = getDbClient(config[type]);
diff --git a/e2e/support/helpers/e2e-qa-databases-helpers.js b/e2e/support/helpers/e2e-qa-databases-helpers.js
index c0bba2b331c..2e453e821ee 100644
--- a/e2e/support/helpers/e2e-qa-databases-helpers.js
+++ b/e2e/support/helpers/e2e-qa-databases-helpers.js
@@ -206,44 +206,7 @@ function recursiveCheckFields(id, i = 0) {
}
export const setupWritableDB = (type = "postgres") => {
- const connectionConfig = {
- postgres: {
- client: "pg",
- connection: {
- ...QA_DB_CREDENTIALS,
- port: QA_POSTGRES_PORT,
- },
- },
- mysql: {
- client: "mysql2",
- connection: {
- ...QA_DB_CREDENTIALS,
- user: "root",
- port: QA_MYSQL_PORT,
- },
- },
- };
-
- const dbName = WRITABLE_DB_CONFIG[type].connection.database;
-
- const dbCheckQuery = {
- postgres: `SELECT FROM pg_database WHERE datname = '${dbName}';`,
- mysql: `SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='${dbName}'`,
- };
-
- // we need to initially connect to the db we know exists to create the writable_db
- cy.task("connectAndQueryDB", {
- connectionConfig: connectionConfig[type],
- query: dbCheckQuery[type],
- }).then((results) => {
- if (!results.rows.length) {
- cy.log(`**-- Adding ${type} DB for actions --**`);
- cy.task("connectAndQueryDB", {
- connectionConfig: connectionConfig[type],
- query: `CREATE DATABASE ${dbName};`,
- });
- }
- });
+ cy.task("createWritableDb", { type });
};
export function queryQADB(query, type = "postgres") {
diff --git a/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.ts b/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.ts
index 2ec1f1a4ac7..277b92660b8 100644
--- a/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.ts
+++ b/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.ts
@@ -103,6 +103,10 @@ function mapDatabricksValues(parsedValues: RegexFields) {
]),
);
+ if (fieldsMap.get("details.additional-options")) {
+ fieldsMap.set("details.advanced-options", true);
+ }
+
return fieldsMap;
}
@@ -127,9 +131,14 @@ function mapMysqlValues(parsedValues: RegexFields) {
["details.ssl", parsedValues.params?.ssl],
[
"details.additional-options",
- objectToString(parsedValues.params, ["ssl"]),
+ objectToString(parsedValues.params, ["ssl"], "&"),
],
]);
+
+ if (fieldsMap.get("details.additional-options")) {
+ fieldsMap.set("details.advanced-options", true);
+ }
+
return fieldsMap;
}
@@ -156,10 +165,14 @@ function mapPostgresValues(parsedValues: RegexFields) {
["details.ssl", parsedValues.params?.ssl],
[
"details.additional-options",
- objectToString(parsedValues.params, ["ssl"]),
+ objectToString(parsedValues.params, ["ssl"], "&"),
],
]);
+ if (fieldsMap.get("details.additional-options")) {
+ fieldsMap.set("details.advanced-options", true);
+ }
+
return fieldsMap;
}
@@ -389,14 +402,16 @@ export function mapFieldsToNestedObject(
}
/**
- * Converts an object to a semicolon-separated string.
+ * Converts an object to a string of key=value pairs.
* @param obj - The object to convert.
* @param filterProperties - The properties to filter out.
- * @returns The semicolon-separated string.
+ * @param separator - The string between pairs.
+ * @returns The joined string.
*/
function objectToString(
obj: Record<string, string | undefined> | undefined,
filterProperties: Array<string> = [],
+ separator = ";",
) {
if (!obj) {
return "";
@@ -405,5 +420,5 @@ function objectToString(
return Object.entries(obj)
.filter(([key]) => !filterProperties.includes(key))
.map(([key, value]) => `${key}=${value}`)
- .join(";");
+ .join(separator);
}
diff --git a/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.unit.spec.ts b/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.unit.spec.ts
index 9737523223f..29edf2b0c3c 100644
--- a/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.unit.spec.ts
+++ b/frontend/src/metabase/databases/components/DatabaseConnectionUri/database-field-mapper.unit.spec.ts
@@ -1,15 +1,78 @@
+import { getSubmitValues } from "metabase/databases/utils/schema";
import { checkNotNull } from "metabase/utils/types";
+import type { EngineKey } from "metabase-types/api";
+import {
+ createMockDatabaseData,
+ createMockEngine,
+} from "metabase-types/api/mocks";
import { mapDatabaseValues } from "./database-field-mapper";
import { parseConnectionUriRegex } from "./parse-connection-regex";
-function mapConnectionString(connectionString: string, engineKey: "redshift") {
+function mapConnectionString(
+ connectionString: string,
+ engineKey: EngineKey,
+): Map<string, string | boolean | undefined> {
const parsedValues = checkNotNull(
parseConnectionUriRegex(connectionString, engineKey),
);
return mapDatabaseValues(parsedValues, engineKey);
}
+describe.each<{
+ engineKey: EngineKey;
+ connectionString: string;
+ additionalOptions: string;
+}>([
+ {
+ engineKey: "mysql",
+ connectionString:
+ "jdbc:mysql://metabase:password@localhost:3304/sample?allowPublicKeyRetrieval=true&useSSL=false",
+ additionalOptions: "allowPublicKeyRetrieval=true&useSSL=false",
+ },
+ {
+ engineKey: "postgres",
+ connectionString:
+ "jdbc:postgresql://metabase:password@localhost:5404/sample?prepareThreshold=0&connectTimeout=10",
+ additionalOptions: "prepareThreshold=0&connectTimeout=10",
+ },
+ {
+ engineKey: "databricks",
+ connectionString:
+ "jdbc:databricks://example.cloud.databricks.com:443;httpPath=/sql/1.0/warehouses/abc;IgnoreTransactions=0",
+ additionalOptions: "IgnoreTransactions=0",
+ },
+])(
+ "mapDatabaseValues - $engineKey",
+ ({ engineKey, connectionString, additionalOptions }) => {
+ it("should preserve additional options in submitted connection details", () => {
+ const fieldsMap = mapConnectionString(connectionString, engineKey);
+ const engine = createMockEngine({
+ "details-fields": [
+ {
+ name: "additional-options",
+ "visible-if": { "advanced-options": true },
+ },
+ ],
+ });
+ const submitted = getSubmitValues(
+ engine,
+ createMockDatabaseData({
+ name: "sample",
+ engine: engineKey,
+ details: {
+ "advanced-options": fieldsMap.get("details.advanced-options"),
+ "additional-options": fieldsMap.get("details.additional-options"),
+ },
+ }),
+ true,
+ );
+
+ expect(submitted.details?.["additional-options"]).toBe(additionalOptions);
+ });
+ },
+);
+
describe("mapDatabaseValues - redshift", () => {
it("should map UID and PWD to dedicated fields and keep unmapped params in additional options", () => {
const fieldsMap = mapConnectionString(