The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GDGT-2144 Require transform target schema on schema-aware DBs (#72220)
Ngoc Khuat64af0de83f0f
diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsxindex 5c691c215c1..da1e1ce17ca 100644--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx@@ -102,6 +102,11 @@ export function CreateTransformModal({[validationSchemaExtension, defaultSchema],);+ const validationContext = useMemo(+ () => ({ supportsSchemas: Boolean(supportsSchemas) }),+ [supportsSchemas],+ );+if (isLoading || error != null) {return <LoadingAndErrorWrapper loading={isLoading} error={error} />;}@@ -119,6 +124,7 @@ export function CreateTransformModal({<FormProviderinitialValues={initialValues}validationSchema={validationSchema}+ validationContext={validationContext}onSubmit={handleSubmit || defaultHandleSubmit}validateOnMount={validateOnMount}>diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.tsindex 8e3e2e3f476..8c3259fd76d 100644--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.ts+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.ts@@ -17,7 +17,14 @@ import {export const VALIDATION_SCHEMA = Yup.object({name: Yup.string().required(Errors.required),targetName: Yup.string().required(Errors.required),- targetSchema: Yup.string().nullable().defined(),+ // `$supportsSchemas` is threaded in via `FormProvider`'s `validationContext`; see `LoginForm.tsx`.+ targetSchema: Yup.string()+ .nullable()+ .defined()+ .when("$supportsSchemas", {+ is: true,+ then: (schema) => schema.required(Errors.required),+ }),collection_id: Yup.number().nullable().defined(),}).concat(INCREMENTAL_TRANSFORM_VALIDATION_SCHEMA);diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.tsnew file mode 100644index 00000000000..0524111d9a9--- /dev/null+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts@@ -0,0 +1,62 @@+import { VALIDATION_SCHEMA } from "./form";++// Full valid form payload; individual tests override only the fields they care about.+const baseValues = {+ name: "My Transform",+ targetName: "my_target_table",+ targetSchema: "public",+ collection_id: null,+ incremental: false,+ sourceStrategy: "checkpoint" as const,+ checkpointFilterFieldId: null,+ targetStrategy: "append" as const,+};++describe("CreateTransformModal VALIDATION_SCHEMA (GDGT-2144)", () => {+ describe("when the database supports schemas", () => {+ const context = { supportsSchemas: true };++ it("accepts a non-blank targetSchema", async () => {+ await expect(+ VALIDATION_SCHEMA.validate(baseValues, { context }),+ ).resolves.toBeTruthy();+ });++ it("rejects a null targetSchema", async () => {+ await expect(+ VALIDATION_SCHEMA.validate(+ { ...baseValues, targetSchema: null },+ { context },+ ),+ ).rejects.toThrow();+ });++ it("rejects an empty-string targetSchema", async () => {+ await expect(+ VALIDATION_SCHEMA.validate(+ { ...baseValues, targetSchema: "" },+ { context },+ ),+ ).rejects.toThrow();+ });+ });++ describe("when the database does not support schemas", () => {+ const context = { supportsSchemas: false };++ it("accepts a null targetSchema", async () => {+ await expect(+ VALIDATION_SCHEMA.validate(+ { ...baseValues, targetSchema: null },+ { context },+ ),+ ).resolves.toBeTruthy();+ });++ it("accepts a non-null targetSchema", async () => {+ await expect(+ VALIDATION_SCHEMA.validate(baseValues, { context }),+ ).resolves.toBeTruthy();+ });+ });+});diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.tsxindex 9d1d9766fea..02498c65128 100644--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.tsx+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.tsx@@ -65,9 +65,15 @@ type EditTransformValues = {schema: string | null;};-const EDIT_TRANSFORM_SCHEMA = Yup.object({+// `$supportsSchemas` is threaded in via `FormProvider`'s `validationContext`; see `LoginForm.tsx`.+export const EDIT_TRANSFORM_SCHEMA = Yup.object({name: Yup.string().required(Errors.required),- schema: Yup.string().nullable(),+ schema: Yup.string()+ .nullable()+ .when("$supportsSchemas", {+ is: true,+ then: (schema) => schema.required(Errors.required),+ }),});type UpdateTargetFormProps = {@@ -106,6 +112,11 @@ function UpdateTargetForm({const error = databaseError ?? schemasError;const supportsSchemas = database && hasFeature(database, "schemas");+ const validationContext = useMemo(+ () => ({ supportsSchemas: Boolean(supportsSchemas) }),+ [supportsSchemas],+ );+if (isLoading || error != null) {return <LoadingAndErrorWrapper loading={isLoading} error={error} />;}@@ -127,6 +138,7 @@ function UpdateTargetForm({<FormProviderinitialValues={initialValues}validationSchema={EDIT_TRANSFORM_SCHEMA}+ validationContext={validationContext}onSubmit={handleSubmit}>{({ dirty }) => (diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.unit.spec.ts b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.unit.spec.tsnew file mode 100644index 00000000000..0c7e113d1e8--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.unit.spec.ts@@ -0,0 +1,44 @@+import { EDIT_TRANSFORM_SCHEMA } from "./UpdateTargetModal";++describe("UpdateTargetModal EDIT_TRANSFORM_SCHEMA (GDGT-2144)", () => {+ describe("when the database supports schemas", () => {+ const context = { supportsSchemas: true };++ it("accepts a non-blank schema", async () => {+ await expect(+ EDIT_TRANSFORM_SCHEMA.validate(+ { name: "t", schema: "public" },+ { context },+ ),+ ).resolves.toBeTruthy();+ });++ it("rejects a null schema", async () => {+ await expect(+ EDIT_TRANSFORM_SCHEMA.validate(+ { name: "t", schema: null },+ { context },+ ),+ ).rejects.toThrow();+ });++ it("rejects an empty-string schema", async () => {+ await expect(+ EDIT_TRANSFORM_SCHEMA.validate({ name: "t", schema: "" }, { context }),+ ).rejects.toThrow();+ });+ });++ describe("when the database does not support schemas", () => {+ const context = { supportsSchemas: false };++ it("accepts a null schema", async () => {+ await expect(+ EDIT_TRANSFORM_SCHEMA.validate(+ { name: "t", schema: null },+ { context },+ ),+ ).resolves.toBeTruthy();+ });+ });+});diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.cljindex 5fc99fb1a70..46abcb35718 100644--- a/src/metabase/driver/postgres.clj+++ b/src/metabase/driver/postgres.clj@@ -317,6 +317,18 @@;; memory in a set like this{:tables (into #{} (describe-syncable-tables database))})+(defn- nullable-in+ "Build a HoneySQL clause that handles nil values in `xs` correctly.+ SQL `IN (NULL)` never matches NULL rows, so when `xs` contains nil we need an explicit `IS NULL` check."+ [column xs]+ (when xs+ (let [non-nil (seq (remove nil? xs))+ has-nil? (some nil? xs)]+ (cond+ (and non-nil has-nil?) [:or [:in column non-nil] [:= column nil]]+ non-nil [:in column non-nil]+ has-nil? [:= column nil]))))+(defmethod sql-jdbc.sync/describe-fields-sql :postgres;; The implementation is based on `getColumns` in https://github.com/pgjdbc/pgjdbc/blob/fcc13e70e6b6bb64b848df4b4ba6b3566b5e95a3/pgjdbc/src/main/java/org/postgresql/jdbc/PgDatabaseMetaData.java[driver & {:keys [schema-names table-names]}]@@ -372,7 +384,7 @@[:= :c.column_name :pk.column_name]]]:where [:and[:raw "c.table_schema !~ '^information_schema|catalog_history|pg_'"]- (when schema-names [:in :c.table_schema schema-names])+ (nullable-in :c.table_schema schema-names)(when table-names [:in :c.table_name table-names])]}{:select [[:pa.attname :name][[:case@@ -399,7 +411,7 @@:where [:and[:= :pc.relkind [:inline "m"]][:>= :pa.attnum [:inline 1]]- (when schema-names [:in :pn.nspname schema-names])+ (nullable-in :pn.nspname schema-names)(when table-names [:in :pc.relname table-names])]}]:order-by [:table-schema :table-name :database-position]}:dialect (sql.qp/quote-style driver)))@@ -1282,8 +1294,11 @@(defmethod driver/create-schema-if-needed! :postgres[driver conn-spec schema]- (let [sql [[(format "CREATE SCHEMA IF NOT EXISTS \"%s\";" schema)]]]- (driver/execute-raw-queries! driver conn-spec sql)))+ ;; Without the blank check, `format` stringifies nil to "null" and creates a schema+ ;; literally named "null" on the target DB.+ (when-not (str/blank? schema)+ (let [sql [[(format "CREATE SCHEMA IF NOT EXISTS \"%s\";" schema)]]]+ (driver/execute-raw-queries! driver conn-spec sql))))(defmethod driver/extra-info :postgres[_driver]diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex 19689a08ac8..212164f5840 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -35,6 +35,7 @@check-database-featurecheck-feature-enabled!validate-incremental-column-type!+ validate-target-schema!validate-transform-query!get-transformsget-transformdiff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.cljindex de7bf8b22c0..498538aa1c9 100644--- a/src/metabase/transforms/crud.clj+++ b/src/metabase/transforms/crud.clj@@ -3,6 +3,7 @@so that non-REST modules (e.g. metabot-v3, workspaces) can use them without dependingon the `-rest` module."(:require+ [clojure.string :as str][metabase.api.common :as api][metabase.database-routing.core :as database-routing][metabase.driver.util :as driver.u]@@ -48,6 +49,18 @@(assoc error:status-code 400)))))+(defn validate-target-schema!+ "Require a non-blank `:target.schema` when the target database supports schemas.++ On schemas-supporting drivers a nil schema makes post-run sync miss the physical table+ and leaves the Metabase table with zero fields."+ [transform]+ (let [db-id (transforms-base.i/target-db-id transform)+ db (t2/select-one :model/Database db-id)]+ (when (and db (driver.u/supports? (:engine db) :schemas db))+ (api/check-400 (not (str/blank? (get-in transform [:target :schema])))+ (deferred-tru "A target schema is required for this database.")))))+(defn validate-incremental-column-type!"Validates that the checkpoint column for an incremental transform has a supported type.@@ -100,6 +113,7 @@([body creator-id](when (transforms-base.u/query-transform? body)(validate-transform-query! body))+ (validate-target-schema! body)(let [creator-id (or creator-id api/*current-user-id*)transform (t2/with-transaction [_](let [tag-ids (:tag_ids body)@@ -134,6 +148,8 @@;; we must validate on a full transform object(check-feature-enabled! new)(check-database-feature new)+ (when (contains? body :target)+ (validate-target-schema! new))(validate-incremental-column-type! new)(when (transforms-base.u/query-transform? old)(validate-transform-query! new)diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 2660253a8f9..bcc47b60fdf 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -1094,6 +1094,43 @@(is (re-find #"CAST" sql))(is (some? (mt/rows (qp/process-query query)))))))))))+(deftest create-schema-if-needed-nil-guard-test+ (testing "create-schema-if-needed! is a no-op when schema is nil or blank (GDGT-2144)"+ (let [executed-queries (atom [])]+ (with-redefs [driver/execute-raw-queries! (fn [_driver _conn-spec queries]+ (swap! executed-queries conj queries))]+ (driver/create-schema-if-needed! :postgres ::fake-conn nil)+ (driver/create-schema-if-needed! :postgres ::fake-conn "")+ (driver/create-schema-if-needed! :postgres ::fake-conn " ")+ (is (empty? @executed-queries)+ "nil/blank schema should not issue any SQL")))))++(deftest ^:parallel describe-fields-sql-nil-schema-test+ (testing "describe-fields-sql for Postgres handles nil schema-names correctly (GDGT-2144)"+ (let [[nil-schema-sql] (sql-jdbc.sync/describe-fields-sql+ :postgres+ {:schema-names [nil]+ :table-names ["my_table"]+ :details {}})+ [mixed-schema-sql] (sql-jdbc.sync/describe-fields-sql+ :postgres+ {:schema-names [nil "public"]+ :table-names ["my_table"]+ :details {}})+ [normal-schema-sql] (sql-jdbc.sync/describe-fields-sql+ :postgres+ {:schema-names ["public"]+ :table-names ["my_table"]+ :details {}})]+ (is (not (re-find #"(?i)IN \(NULL\)" nil-schema-sql))+ "rendered SQL must not contain `IN (NULL)` which never matches anything")+ (is (re-find #"\"table_schema\" IS NULL" nil-schema-sql)+ "passing [nil] schemas should produce an IS NULL check on table_schema")+ (is (re-find #"\"table_schema\" IN .+OR .+\"table_schema\" IS NULL" mixed-schema-sql)+ "mixed nil + non-nil schemas should include both IN and IS NULL")+ (is (not (re-find #"\"table_schema\" IS NULL" normal-schema-sql))+ "non-nil-only schemas should not have IS NULL on table_schema"))))+;; API tests are in [[metabase.actions-rest.api-test]](deftest ^:parallel actions-maybe-parse-sql-violate-not-null-constraint-test(testing "violate not null constraint"diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex c2b58350d6a..2bd6f5a9e19 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -351,6 +351,18 @@(is (false? (:is_writable table))"Computed transform tables should have is_writable=false")))))))+(deftest target-table-handles-nil-schema-test+ (testing "target-table can find a Table row with schema=nil (GDGT-2144)"+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}+ :model/Table _table {:db_id db-id+ :schema nil+ :name "some_nil_schema_table"+ :active true}]+ (let [found (transforms-base.u/target-table db-id {:schema nil :name "some_nil_schema_table"})]+ (is (some? found)+ "target-table must find a Table row with schema=nil — otherwise schemas-less drivers (MySQL) would silently miss their targets")+ (is (= "some_nil_schema_table" (:name found)))))))+(deftest execute-sets-transform-id-on-target-table-test(testing "Executing a query transform sets transform_id on the target table"(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljindex 45ccba08554..a5f1b882223 100644--- a/test/metabase/transforms_rest/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -3,6 +3,8 @@(:require[clojure.test :refer :all][medley.core :as m]+ [metabase.driver :as driver]+ [metabase.driver.util :as driver.u][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.permissions.core :as perms]@@ -92,6 +94,59 @@(is (map? (:owner response)))(is (= lucky-id (get-in response [:owner :id]))))))))))))+(deftest update-transform-without-schema-test+ (testing "Updating a transform to clear its schema is rejected on schemas-supporting databases"+ (mt/with-premium-features #{}+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/dataset transforms-dataset/transforms-test+ (when (driver.u/supports? driver/*driver* :schemas (mt/db))+ (with-transform-cleanup! [table-name "update_schema_products"]+ (let [query (make-query "Gadget")+ schema (get-test-schema)+ created (mt/user-http-request :crowberto :post 200 "transform"+ {:name "Update Schema Products"+ :source {:type "query"+ :query query}+ :target {:type "table"+ :schema schema+ :name table-name}})+ transform-id (:id created)]+ (testing "PUT with a nil schema in target is rejected"+ (mt/user-http-request :crowberto :put 400+ (format "transform/%s" transform-id)+ {:target {:type "table"+ :schema nil+ :name table-name}}))+ (testing "PUT with a non-blank schema still succeeds"+ (let [updated (mt/user-http-request :crowberto :put 200+ (format "transform/%s" transform-id)+ {:name "Renamed"})]+ (is (= "Renamed" (:name updated)))))))))))))++(deftest create-transform-without-schema-test+ (testing "Creating a transform without a schema is rejected on databases that support schemas, and allowed on those that don't"+ (mt/with-premium-features #{}+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/dataset transforms-dataset/transforms-test+ (mt/with-data-analyst-role! (mt/user->id :lucky)+ (mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes+ (with-transform-cleanup! [table-name "no_schema_products"]+ (let [query (make-query "Gadget")+ request (fn [schema]+ {:name "No Schema Products"+ :source {:type "query"+ :query query}+ :target {:type "table"+ :schema schema+ :name table-name}})]+ (if (driver.u/supports? driver/*driver* :schemas (mt/db))+ (testing "nil schema is rejected with 400 on schemas-supporting driver"+ (let [response (mt/user-http-request :lucky :post 400 "transform" (request nil))]+ (is (nil? (:id response)))))+ (testing "nil schema is allowed on non-schemas drivers"+ (let [response (mt/user-http-request :lucky :post 200 "transform" (request nil))]+ (is (some? (:id response)))))))))))))))+(deftest create-transform-with-param-test(mt/with-premium-features #{}(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)