The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Restructure Database form to handle engine change internally (#78932)
Nick Fitzpatrick33b58efb9fa8
diff --git a/frontend/src/metabase-types/api/database.ts b/frontend/src/metabase-types/api/database.tsindex 91b890098bb..b535fb44c40 100644--- a/frontend/src/metabase-types/api/database.ts+++ b/frontend/src/metabase-types/api/database.ts@@ -94,6 +94,8 @@ export interface DatabaseData {id?: DatabaseId;name: string;engine: string | undefined;+ /** Hosting provider detected from the connection details, e.g. "AWS RDS" */+ provider_name?: string | null;// If current user lacks write permission to database, `details` will be// missing in responses from the backend, cf. implementation of// [[metabase.models.interface/to-json]] for `:model/Database`:diff --git a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsxindex 3193b626705..f4c9442010c 100644--- a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx+++ b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseForm.tsx@@ -1,8 +1,9 @@-import { useCallback, useEffect, useMemo, useState } from "react";+import { useCallback, useMemo } from "react";+import _ from "underscore";import { Form, FormProvider } from "metabase/forms";import { useSelector } from "metabase/redux";-import type { DatabaseData, EngineKey } from "metabase-types/api";+import type { DatabaseData } from "metabase-types/api";import { getEngines } from "../../selectors";import type { FormLocation } from "../../types";@@ -11,7 +12,7 @@ import { getSubmitValues, getValidationSchema } from "../../utils/schema";import { DatabaseFormBody } from "./DatabaseFormBody";import { DatabaseFormFooter } from "./DatabaseFormFooter";import { FormDirtyStateProvider } from "./context";-import { getEngine, getEngineKey } from "./utils";+import { castEngineValues, getEngine, getEngineKey } from "./utils";export type EngineFieldState = "default" | "hidden" | "disabled";@@ -67,38 +68,33 @@ export const DatabaseForm = ({const initialEngineKey = useMemo(() => {return getEngineKey(engines, initialData, isAdvanced);}, [engines, initialData, isAdvanced]);- const [engineKey, setEngineKey] = useState(initialEngineKey);- const engine = getEngine(engines, engineKey);- const validationSchema = useMemo(() => {- return getValidationSchema(engine, engineKey, isAdvanced);- }, [engine, engineKey, isAdvanced]);+ const getSchema = useMemo(() => {+ return _.memoize((engineKey: string | undefined) =>+ getValidationSchema(getEngine(engines, engineKey), engineKey, isAdvanced),+ );+ }, [engines, isAdvanced]);++ // The form's shape depends on the engine, so the schema has to follow `values.engine`.+ const validationSchema = useCallback(+ (values: DatabaseData) => getSchema(values.engine),+ [getSchema],+ );const initialValues = useMemo(() => {- return validationSchema.cast(- { ...initialData, engine: engineKey },- { stripUnknown: true },+ return castEngineValues(+ engines,+ { ...initialData, engine: initialEngineKey },+ isAdvanced,);- }, [initialData, engineKey, validationSchema]);-- useEffect(() => {- // during page load, if initialData changes, initialEngineKey will also change- setEngineKey(initialEngineKey);- }, [initialEngineKey]);+ }, [engines, initialData, initialEngineKey, isAdvanced]);const handleSubmit = useCallback((values: DatabaseData) => {+ const engine = getEngine(engines, values.engine);return onSubmit?.(getSubmitValues(engine, values, isAdvanced));},- [engine, isAdvanced, onSubmit],- );-- const handleEngineChange = useCallback(- (engineKey: string | undefined) => {- setEngineKey(engineKey);- onEngineChange?.(engineKey);- },- [onEngineChange],+ [engines, isAdvanced, onSubmit],);return (@@ -119,13 +115,10 @@ export const DatabaseForm = ({><FormDirtyStateProvider onDirtyStateChange={onDirtyStateChange}><DatabaseFormBody- engine={engine}- // casting won't be needed after migrating all usages of engineKey- engineKey={engineKey as EngineKey}engines={engines}autofocusFieldName={autofocusFieldName}isAdvanced={isAdvanced}- onEngineChange={handleEngineChange}+ onEngineChange={onEngineChange}config={config}showSampleDatabase={showSampleDatabase}location={location}diff --git a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsxindex 52a8453972a..cb52c5692a0 100644--- a/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx+++ b/frontend/src/metabase/databases/components/DatabaseForm/DatabaseFormBody.tsx@@ -1,5 +1,5 @@import { useFormikContext } from "formik";-import type { JSX } from "react";+import { type JSX, useCallback } from "react";import { match } from "ts-pattern";import type {@@ -16,23 +16,24 @@ import { DatabaseFormError } from "../DatabaseFormError";import { DatabaseNameField } from "../DatabaseNameField";import { DatabaseFormBodyDetails } from "./DatabaseFormBodyDetails";-import { useHasConnectionError } from "./utils";+import {+ getEngine,+ getEngineDefaults,+ mergeRetainedValues,+ useHasConnectionError,+} from "./utils";interface DatabaseFormBodyProps {- engine: Engine | undefined;- engineKey: EngineKey | undefined;engines: Record<string, Engine>;autofocusFieldName?: string;isAdvanced: boolean;- onEngineChange: (engineKey: string | undefined) => void;+ onEngineChange?: (engineKey: string | undefined) => void;config: DatabaseFormConfig;showSampleDatabase?: boolean;location: FormLocation;}export const DatabaseFormBody = ({- engine,- engineKey,engines,autofocusFieldName,isAdvanced,@@ -41,10 +42,28 @@ export const DatabaseFormBody = ({showSampleDatabase = false,location,}: DatabaseFormBodyProps): JSX.Element => {- const { setValues } = useFormikContext<DatabaseData>();+ const { setValues, values } = useFormikContext<DatabaseData>();const hasConnectionError = useHasConnectionError();const { engine: engineFieldConfig, name: nameFieldConfig } = config;+ // casting won't be needed after migrating all usages of engineKey+ const engineKey = values.engine as EngineKey | undefined;+ const engine = getEngine(engines, engineKey);++ const handleEngineChange = useCallback(+ (nextEngineKey: string | undefined) => {+ setValues((previousValues) =>+ mergeRetainedValues(+ previousValues,+ getEngineDefaults(engines, nextEngineKey, isAdvanced),+ getEngine(engines, nextEngineKey),+ ),+ );+ onEngineChange?.(nextEngineKey);+ },+ [engines, isAdvanced, onEngineChange, setValues],+ );+const px = match(location).with("setup", () => "sm").with("embedding_setup", () => "xl")@@ -61,14 +80,14 @@ export const DatabaseFormBody = ({engineKey={engineKey}engines={engines}isAdvanced={isAdvanced}- onChange={onEngineChange}+ onChange={handleEngineChange}disabled={engineFieldConfig?.fieldState === "disabled"}showSampleDatabase={showSampleDatabase}/><DatabaseEngineWarningengineKey={engineKey}engines={engines}- onChange={onEngineChange}+ onChange={handleEngineChange}/></>)}diff --git a/frontend/src/metabase/databases/components/DatabaseForm/tests/engine-change.unit.spec.tsx b/frontend/src/metabase/databases/components/DatabaseForm/tests/engine-change.unit.spec.tsxnew file mode 100644index 00000000000..fe250f5278b--- /dev/null+++ b/frontend/src/metabase/databases/components/DatabaseForm/tests/engine-change.unit.spec.tsx@@ -0,0 +1,153 @@+import userEvent from "@testing-library/user-event";+import { useState } from "react";++import { mockSettings } from "__support__/settings";+import { renderWithProviders, screen, waitFor } from "__support__/ui";+import { createMockState } from "metabase/redux/store/mocks";+import type { DatabaseData, Engine } from "metabase-types/api";++import { DatabaseForm } from "../DatabaseForm";++import { mysqlFormConfig } from "./mysql-form-config.mock";+import { TEST_ENGINES, setup } from "./setup";++const ENGINES: Record<string, Engine> = {+ ...TEST_ENGINES,+ mysql: mysqlFormConfig,+};++const selectDatabaseType = async (driverName: string) => {+ await userEvent.click(screen.getByLabelText("Database type"));+ await userEvent.click(+ await screen.findByRole("option", { name: driverName }),+ );+ expect(await screen.findByDisplayValue(driverName)).toBeInTheDocument();+};++describe("DatabaseForm engine change", () => {+ it("should keep the values that already have been entered (metabase#74569)", async () => {+ setup({ engines: ENGINES, initialValues: { engine: "postgres" } });++ await userEvent.type(screen.getByLabelText("Display name"), "My database");+ await userEvent.type(screen.getByLabelText("Host"), "localhost");+ await userEvent.type(screen.getByLabelText("Database name"), "birds");+ await userEvent.type(screen.getByLabelText("Username"), "bird-watcher");+ await userEvent.type(screen.getByLabelText("Password"), "hunter2");++ await selectDatabaseType("MySQL");++ expect(screen.getByLabelText("Display name")).toHaveValue("My database");+ expect(screen.getByLabelText("Host")).toHaveValue("localhost");+ expect(screen.getByLabelText("Database name")).toHaveValue("birds");+ expect(screen.getByLabelText("Username")).toHaveValue("bird-watcher");+ expect(screen.getByLabelText("Password")).toHaveValue("hunter2");+ });++ it("should submit the values that were entered before the engine changed (metabase#74569)", async () => {+ const { onSubmit } = setup({+ engines: ENGINES,+ initialValues: { engine: "postgres" },+ });++ await userEvent.type(screen.getByLabelText("Display name"), "My database");+ await userEvent.type(screen.getByLabelText("Host"), "localhost");+ await userEvent.type(screen.getByLabelText("Database name"), "birds");+ await userEvent.type(screen.getByLabelText("Username"), "bird-watcher");++ await selectDatabaseType("MySQL");++ const saveButton = screen.getByRole("button", { name: "Save" });+ await waitFor(() => expect(saveButton).toBeEnabled());+ await userEvent.click(saveButton);++ await waitFor(() => {+ expect(onSubmit).toHaveBeenCalledWith(+ expect.objectContaining({+ engine: "mysql",+ name: "My database",+ details: expect.objectContaining({+ host: "localhost",+ dbname: "birds",+ user: "bird-watcher",+ ssl: false,+ }),+ }),+ );+ });+ });++ it("should discard the details that the newly selected engine does not have", async () => {+ setup({ engines: ENGINES, initialValues: { engine: "postgres" } });++ await userEvent.type(screen.getByLabelText("Host"), "localhost");++ await selectDatabaseType("H2");++ expect(screen.queryByLabelText("Host")).not.toBeInTheDocument();+ expect(screen.getByLabelText("Connection String")).toHaveValue("");+ });++ it("should follow the engine of the initial data when it arrives after the first render", async () => {+ // the edit page renders the form before the database record has loaded+ const LateLoadingForm = () => {+ const [initialValues, setInitialValues] = useState<Partial<DatabaseData>>(+ {},+ );+ const load = () =>+ setInitialValues({+ engine: "mysql",+ name: "Loaded database",+ details: { host: "db.example.com", dbname: "birds" },+ });++ return (+ <>+ <button onClick={load}>Load</button>+ <DatabaseForm+ initialValues={initialValues}+ config={{ isAdvanced: true }}+ location="admin"+ />+ </>+ );+ };++ renderWithProviders(<LateLoadingForm />, {+ storeInitialState: createMockState({+ settings: mockSettings({ engines: ENGINES }),+ }),+ });++ expect(await screen.findByDisplayValue("PostgreSQL")).toBeInTheDocument();++ await userEvent.click(screen.getByRole("button", { name: "Load" }));++ expect(await screen.findByDisplayValue("MySQL")).toBeInTheDocument();+ expect(screen.getByLabelText("Display name")).toHaveValue(+ "Loaded database",+ );+ expect(screen.getByLabelText("Host")).toHaveValue("db.example.com");+ expect(screen.getByLabelText("Database name")).toHaveValue("birds");+ });++ it("should discard the connection string, which is engine specific", async () => {+ setup({ engines: ENGINES, initialValues: { engine: "postgres" } });++ const connectionStringInput = () =>+ screen.getByLabelText("Connection string (optional)");++ await userEvent.type(+ connectionStringInput(),+ "jdbc:postgresql://user:pass@localhost:5432/mydb",+ );+ await waitFor(() =>+ expect(screen.getByLabelText("Database name")).toHaveValue("mydb"),+ );++ await selectDatabaseType("MySQL");++ expect(connectionStringInput()).toHaveValue("");+ // the details it pre-filled are still kept, only the string itself is reset+ expect(screen.getByLabelText("Database name")).toHaveValue("mydb");+ });+});diff --git a/frontend/src/metabase/databases/components/DatabaseForm/tests/mysql-form-config.mock.ts b/frontend/src/metabase/databases/components/DatabaseForm/tests/mysql-form-config.mock.tsnew file mode 100644index 00000000000..2c13956e4c7--- /dev/null+++ b/frontend/src/metabase/databases/components/DatabaseForm/tests/mysql-form-config.mock.ts@@ -0,0 +1,54 @@+import type { Engine } from "metabase-types/api";++export const mysqlFormConfig = {+ source: {+ type: "official",+ contact: null,+ },+ "details-fields": [+ {+ type: "group",+ "container-style": ["grid", "3fr 1fr"],+ fields: [+ {+ name: "host",+ "display-name": "Host",+ placeholder: "name.database.com",+ },+ {+ name: "port",+ "display-name": "Port",+ type: "integer",+ placeholder: 3306,+ },+ ],+ },+ {+ name: "dbname",+ "display-name": "Database name",+ placeholder: "birds_of_the_world",+ required: true,+ },+ {+ name: "user",+ "display-name": "Username",+ placeholder: "username",+ required: true,+ },+ {+ name: "password",+ "display-name": "Password",+ type: "password",+ placeholder: "••••••••",+ },+ {+ name: "ssl",+ "display-name": "Use a secure connection (SSL)",+ type: "boolean",+ default: false,+ },+ ],+ "driver-name": "MySQL",+ "superseded-by": null,+ "extra-info": null,+} satisfies Engine;diff --git a/frontend/src/metabase/databases/components/DatabaseForm/utils.ts b/frontend/src/metabase/databases/components/DatabaseForm/utils.tsindex 998a3451403..db454416188 100644--- a/frontend/src/metabase/databases/components/DatabaseForm/utils.ts+++ b/frontend/src/metabase/databases/components/DatabaseForm/utils.ts@@ -1,8 +1,15 @@import { useContext } from "react";+import { match } from "ts-pattern";+import _ from "underscore";+import type { DatabaseFormValues } from "metabase/databases/types";import { getDefaultEngineKey } from "metabase/databases/utils/engine";+import {+ getFlattenedFields,+ getValidationSchema,+} from "metabase/databases/utils/schema";import { useFormErrorMessage } from "metabase/forms";-import type { DatabaseData, Engine } from "metabase-types/api";+import type { DatabaseData, Engine, EngineField } from "metabase-types/api";import { FormDirtyStateContext } from "./context";@@ -33,3 +40,80 @@ export const getEngineKey = (export const useIsFormDirty = () => {return useContext(FormDirtyStateContext);};++export const castEngineValues = (+ engines: Record<string, Engine>,+ values: Partial<DatabaseData>,+ isAdvanced: boolean,+): DatabaseData => {+ const engineKey = values.engine;+ const schema = getValidationSchema(+ getEngine(engines, engineKey),+ engineKey,+ isAdvanced,+ );+ return schema.cast(values, { stripUnknown: true });+};++export const getEngineDefaults = (+ engines: Record<string, Engine>,+ engineKey: string | undefined,+ isAdvanced: boolean,+): DatabaseData => castEngineValues(engines, { engine: engineKey }, isAdvanced);++/**+ * Values that describe the engine that was selected rather than the connection itself,+ * so they cannot be carried over to another engine.+ */+const ENGINE_SPECIFIC_KEYS: (keyof DatabaseFormValues)[] = [+ "engine",+ "connection-string",+ "provider_name",+];++export const mergeRetainedValues = (+ previousValues: DatabaseData,+ defaultValues: DatabaseData,+ engine: Engine | undefined,+): DatabaseData => {+ const fields: Record<string, EngineField | undefined> = _.indexBy(+ getFlattenedFields(engine?.["details-fields"] ?? []),+ "name",+ );++ const details = _.mapObject(+ defaultValues.details ?? {},+ (defaultValue, name) => {+ const value = previousValues.details?.[name];+ return canRetainDetail(fields[name], value) ? value : defaultValue;+ },+ );++ return {+ ...defaultValues,+ ..._.omit(previousValues, ENGINE_SPECIFIC_KEYS),+ details,+ };+};++const canRetainDetail = (field: EngineField | undefined, value: unknown) => {+ if (field == null || value == null) {+ return false;+ }++ return match(field.type)+ .with("hidden", () => false)+ .with("integer", () => typeof value === "number")+ .with("boolean", "section", () => typeof value === "boolean")+ .with(+ "string",+ "password",+ "text",+ "select",+ "textFile",+ "info",+ undefined,+ () => typeof value === "string",+ )+ .exhaustive();+};diff --git a/frontend/src/metabase/databases/components/DatabaseForm/utils.unit.spec.ts b/frontend/src/metabase/databases/components/DatabaseForm/utils.unit.spec.tsnew file mode 100644index 00000000000..10ca0665815--- /dev/null+++ b/frontend/src/metabase/databases/components/DatabaseForm/utils.unit.spec.ts@@ -0,0 +1,122 @@+import type { Engine } from "metabase-types/api";+import { createMockDatabaseData } from "metabase-types/api/mocks";++import { mergeRetainedValues } from "./utils";++const ENGINE: Engine = {+ source: { type: "official", contact: null },+ "driver-name": "PostgreSQL",+ "superseded-by": null,+ "extra-info": null,+ "details-fields": [+ { name: "host" },+ { name: "port", type: "integer" },+ { name: "ssl", type: "boolean" },+ { name: "region", type: "select" },+ { name: "is-destination-database", type: "hidden" },+ ],+};++const DEFAULT_VALUES = {+ ...createMockDatabaseData({+ engine: "postgres",+ name: "",+ details: {+ host: null,+ port: null,+ ssl: false,+ region: null,+ "is-destination-database": false,+ },+ }),+ "connection-string": "",+ provider_name: null,+};++describe("mergeRetainedValues", () => {+ it("should keep the details that the engine declares with the same type", () => {+ const previousValues = createMockDatabaseData({+ engine: "mysql",+ name: "My database",+ details: {+ host: "localhost",+ port: 5432,+ ssl: true,+ region: "us-east-1",+ },+ });++ expect(+ mergeRetainedValues(previousValues, DEFAULT_VALUES, ENGINE),+ ).toMatchObject({+ name: "My database",+ details: {+ host: "localhost",+ port: 5432,+ ssl: true,+ region: "us-east-1",+ },+ });+ });++ it("should discard the details that the engine does not declare", () => {+ const previousValues = createMockDatabaseData({+ details: { host: "localhost", dbname: "birds" },+ });++ const { details } = mergeRetainedValues(+ previousValues,+ DEFAULT_VALUES,+ ENGINE,+ );++ expect(details).toHaveProperty("host", "localhost");+ expect(details).not.toHaveProperty("dbname");+ });++ it("should fall back to the default when the engine declares another type", () => {+ const previousValues = createMockDatabaseData({+ details: { host: 5432, port: "5432", ssl: "yes" },+ });++ expect(+ mergeRetainedValues(previousValues, DEFAULT_VALUES, ENGINE).details,+ ).toEqual(DEFAULT_VALUES.details);+ });++ it("should not carry over hidden details, which the engine controls", () => {+ const previousValues = createMockDatabaseData({+ details: { "is-destination-database": true },+ });++ expect(+ mergeRetainedValues(previousValues, DEFAULT_VALUES, ENGINE).details,+ ).toHaveProperty("is-destination-database", false);+ });++ it("should reset the values that describe the engine that was selected", () => {+ const previousValues = {+ ...createMockDatabaseData({ engine: "mysql" }),+ "connection-string": "jdbc:mysql://localhost:3306/birds",+ provider_name: "Neon",+ };++ expect(mergeRetainedValues(previousValues, DEFAULT_VALUES, ENGINE)).toEqual(+ expect.objectContaining({+ engine: "postgres",+ "connection-string": "",+ provider_name: null,+ }),+ );+ });++ it("should fall back to the defaults when there is no engine to check against", () => {+ const previousValues = createMockDatabaseData({+ details: { host: "localhost" },+ });++ expect(+ mergeRetainedValues(previousValues, DEFAULT_VALUES, undefined).details,+ ).toEqual(DEFAULT_VALUES.details);+ });+});diff --git a/frontend/src/metabase/databases/types.ts b/frontend/src/metabase/databases/types.tsindex 2cbefd96700..80ef1fe7334 100644--- a/frontend/src/metabase/databases/types.ts+++ b/frontend/src/metabase/databases/types.ts@@ -1,6 +1,18 @@import type { ComponentType, JSX, ReactNode } from "react";-import type { EngineFieldOption, EngineFieldType } from "metabase-types/api";+import type {+ DatabaseData,+ EngineFieldOption,+ EngineFieldType,+} from "metabase-types/api";++/**+ * `connection-string` is a FE only field used to prefill the form, so it is not+ * part of the database itself.+ */+export type DatabaseFormValues = DatabaseData & {+ "connection-string"?: string;+};export interface EngineOption {name: string;diff --git a/frontend/src/metabase/forms/components/FormProvider/FormProvider.tsx b/frontend/src/metabase/forms/components/FormProvider/FormProvider.tsxindex 3e985faff64..bb9c03f960a 100644--- a/frontend/src/metabase/forms/components/FormProvider/FormProvider.tsx+++ b/frontend/src/metabase/forms/components/FormProvider/FormProvider.tsx@@ -1,14 +1,14 @@import type { FormikConfig, FormikValues } from "formik";import { Formik } from "formik";import { useMemo } from "react";-import type { AnySchema } from "yup";import type { FormStatus } from "../../contexts";import { FormContext } from "../../contexts";+import type { ValidationSchema } from "../../hooks";import { useFormSubmit, useFormValidation } from "../../hooks";export interface FormProviderProps<T, C> extends FormikConfig<T> {- validationSchema?: AnySchema;+ validationSchema?: ValidationSchema<T>;validationContext?: C;}diff --git a/frontend/src/metabase/forms/hooks/use-form-validation/use-form-validation.ts b/frontend/src/metabase/forms/hooks/use-form-validation/use-form-validation.tsindex 26deb61f2c1..5fa6ea52fbf 100644--- a/frontend/src/metabase/forms/hooks/use-form-validation/use-form-validation.ts+++ b/frontend/src/metabase/forms/hooks/use-form-validation/use-form-validation.ts@@ -3,9 +3,11 @@ import { prepareDataForValidation, yupToFormErrors } from "formik";import { useCallback, useMemo } from "react";import type { AnySchema } from "yup";+export type ValidationSchema<T> = AnySchema | ((values: T) => AnySchema);+export interface UseFormValidationProps<T extends FormikValues, C> {initialValues: T;- validationSchema?: AnySchema;+ validationSchema?: ValidationSchema<T>;validationContext?: C;}@@ -23,16 +25,20 @@ export const useFormValidation = <T extends FormikValues, C>({if (validationSchema) {return validateSchemaInitial(initialValues,- validationSchema,+ resolveSchema(validationSchema, initialValues),validationContext,);}}, [initialValues, validationSchema, validationContext]);const handleValidate = useCallback(- (values: FormikValues) => {+ (values: T) => {if (validationSchema) {- return validateSchema(values, validationSchema, validationContext);+ return validateSchema(+ values,+ resolveSchema(validationSchema, values),+ validationContext,+ );}},[validationSchema, validationContext],@@ -44,6 +50,9 @@ export const useFormValidation = <T extends FormikValues, C>({};};+const resolveSchema = <T>(schema: ValidationSchema<T>, values: T) =>+ typeof schema === "function" ? schema(values) : schema;+const validateSchema = async <T extends FormikValues, C>(values: T,validationSchema: AnySchema,