Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GDGT-2144 Require transform target schema on schema-aware DBs (#72220)

Ngoc Khuat64af0de83f0f
diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
index 5c691c215c1..da1e1ce17ca 100644
--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
@@ -102,6 +102,11 @@ export function CreateTransformModal({
[validationSchemaExtension, defaultSchema],
);
+ const validationContext = useMemo(
+ () => ({ supportsSchemas: Boolean(supportsSchemas) }),
+ [supportsSchemas],
+ );
+
if (isLoading || error != null) {
return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
}
@@ -119,6 +124,7 @@ export function CreateTransformModal({
<FormProvider
initialValues={initialValues}
validationSchema={validationSchema}
+ validationContext={validationContext}
onSubmit={handleSubmit || defaultHandleSubmit}
validateOnMount={validateOnMount}
>
diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.ts
index 8e3e2e3f476..8c3259fd76d 100644
--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.ts
+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.ts
@@ -17,7 +17,14 @@ import {
export const VALIDATION_SCHEMA = Yup.object({
name: Yup.string().required(Errors.required),
targetName: Yup.string().required(Errors.required),
- targetSchema: Yup.string().nullable().defined(),
+ // `$supportsSchemas` is threaded in via `FormProvider`'s `validationContext`; see `LoginForm.tsx`.
+ targetSchema: Yup.string()
+ .nullable()
+ .defined()
+ .when("$supportsSchemas", {
+ is: true,
+ then: (schema) => schema.required(Errors.required),
+ }),
collection_id: Yup.number().nullable().defined(),
}).concat(INCREMENTAL_TRANSFORM_VALIDATION_SCHEMA);
diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts
new file mode 100644
index 00000000000..0524111d9a9
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts
@@ -0,0 +1,62 @@
+import { VALIDATION_SCHEMA } from "./form";
+
+// Full valid form payload; individual tests override only the fields they care about.
+const baseValues = {
+ name: "My Transform",
+ targetName: "my_target_table",
+ targetSchema: "public",
+ collection_id: null,
+ incremental: false,
+ sourceStrategy: "checkpoint" as const,
+ checkpointFilterFieldId: null,
+ targetStrategy: "append" as const,
+};
+
+describe("CreateTransformModal VALIDATION_SCHEMA (GDGT-2144)", () => {
+ describe("when the database supports schemas", () => {
+ const context = { supportsSchemas: true };
+
+ it("accepts a non-blank targetSchema", async () => {
+ await expect(
+ VALIDATION_SCHEMA.validate(baseValues, { context }),
+ ).resolves.toBeTruthy();
+ });
+
+ it("rejects a null targetSchema", async () => {
+ await expect(
+ VALIDATION_SCHEMA.validate(
+ { ...baseValues, targetSchema: null },
+ { context },
+ ),
+ ).rejects.toThrow();
+ });
+
+ it("rejects an empty-string targetSchema", async () => {
+ await expect(
+ VALIDATION_SCHEMA.validate(
+ { ...baseValues, targetSchema: "" },
+ { context },
+ ),
+ ).rejects.toThrow();
+ });
+ });
+
+ describe("when the database does not support schemas", () => {
+ const context = { supportsSchemas: false };
+
+ it("accepts a null targetSchema", async () => {
+ await expect(
+ VALIDATION_SCHEMA.validate(
+ { ...baseValues, targetSchema: null },
+ { context },
+ ),
+ ).resolves.toBeTruthy();
+ });
+
+ it("accepts a non-null targetSchema", async () => {
+ await expect(
+ VALIDATION_SCHEMA.validate(baseValues, { context }),
+ ).resolves.toBeTruthy();
+ });
+ });
+});
diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.tsx
index 9d1d9766fea..02498c65128 100644
--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.tsx
@@ -65,9 +65,15 @@ type EditTransformValues = {
schema: string | null;
};
-const EDIT_TRANSFORM_SCHEMA = Yup.object({
+// `$supportsSchemas` is threaded in via `FormProvider`'s `validationContext`; see `LoginForm.tsx`.
+export const EDIT_TRANSFORM_SCHEMA = Yup.object({
name: Yup.string().required(Errors.required),
- schema: Yup.string().nullable(),
+ schema: Yup.string()
+ .nullable()
+ .when("$supportsSchemas", {
+ is: true,
+ then: (schema) => schema.required(Errors.required),
+ }),
});
type UpdateTargetFormProps = {
@@ -106,6 +112,11 @@ function UpdateTargetForm({
const error = databaseError ?? schemasError;
const supportsSchemas = database && hasFeature(database, "schemas");
+ const validationContext = useMemo(
+ () => ({ supportsSchemas: Boolean(supportsSchemas) }),
+ [supportsSchemas],
+ );
+
if (isLoading || error != null) {
return <LoadingAndErrorWrapper loading={isLoading} error={error} />;
}
@@ -127,6 +138,7 @@ function UpdateTargetForm({
<FormProvider
initialValues={initialValues}
validationSchema={EDIT_TRANSFORM_SCHEMA}
+ validationContext={validationContext}
onSubmit={handleSubmit}
>
{({ dirty }) => (
diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.unit.spec.ts b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.unit.spec.ts
new file mode 100644
index 00000000000..0c7e113d1e8
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateTargetModal/UpdateTargetModal.unit.spec.ts
@@ -0,0 +1,44 @@
+import { EDIT_TRANSFORM_SCHEMA } from "./UpdateTargetModal";
+
+describe("UpdateTargetModal EDIT_TRANSFORM_SCHEMA (GDGT-2144)", () => {
+ describe("when the database supports schemas", () => {
+ const context = { supportsSchemas: true };
+
+ it("accepts a non-blank schema", async () => {
+ await expect(
+ EDIT_TRANSFORM_SCHEMA.validate(
+ { name: "t", schema: "public" },
+ { context },
+ ),
+ ).resolves.toBeTruthy();
+ });
+
+ it("rejects a null schema", async () => {
+ await expect(
+ EDIT_TRANSFORM_SCHEMA.validate(
+ { name: "t", schema: null },
+ { context },
+ ),
+ ).rejects.toThrow();
+ });
+
+ it("rejects an empty-string schema", async () => {
+ await expect(
+ EDIT_TRANSFORM_SCHEMA.validate({ name: "t", schema: "" }, { context }),
+ ).rejects.toThrow();
+ });
+ });
+
+ describe("when the database does not support schemas", () => {
+ const context = { supportsSchemas: false };
+
+ it("accepts a null schema", async () => {
+ await expect(
+ EDIT_TRANSFORM_SCHEMA.validate(
+ { name: "t", schema: null },
+ { context },
+ ),
+ ).resolves.toBeTruthy();
+ });
+ });
+});
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index 5fc99fb1a70..46abcb35718 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -317,6 +317,18 @@
;; memory in a set like this
{:tables (into #{} (describe-syncable-tables database))})
+(defn- nullable-in
+ "Build a HoneySQL clause that handles nil values in `xs` correctly.
+ SQL `IN (NULL)` never matches NULL rows, so when `xs` contains nil we need an explicit `IS NULL` check."
+ [column xs]
+ (when xs
+ (let [non-nil (seq (remove nil? xs))
+ has-nil? (some nil? xs)]
+ (cond
+ (and non-nil has-nil?) [:or [:in column non-nil] [:= column nil]]
+ non-nil [:in column non-nil]
+ has-nil? [:= column nil]))))
+
(defmethod sql-jdbc.sync/describe-fields-sql :postgres
;; The implementation is based on `getColumns` in https://github.com/pgjdbc/pgjdbc/blob/fcc13e70e6b6bb64b848df4b4ba6b3566b5e95a3/pgjdbc/src/main/java/org/postgresql/jdbc/PgDatabaseMetaData.java
[driver & {:keys [schema-names table-names]}]
@@ -372,7 +384,7 @@
[:= :c.column_name :pk.column_name]]]
:where [:and
[:raw "c.table_schema !~ '^information_schema|catalog_history|pg_'"]
- (when schema-names [:in :c.table_schema schema-names])
+ (nullable-in :c.table_schema schema-names)
(when table-names [:in :c.table_name table-names])]}
{:select [[:pa.attname :name]
[[:case
@@ -399,7 +411,7 @@
:where [:and
[:= :pc.relkind [:inline "m"]]
[:>= :pa.attnum [:inline 1]]
- (when schema-names [:in :pn.nspname schema-names])
+ (nullable-in :pn.nspname schema-names)
(when table-names [:in :pc.relname table-names])]}]
:order-by [:table-schema :table-name :database-position]}
:dialect (sql.qp/quote-style driver)))
@@ -1282,8 +1294,11 @@
(defmethod driver/create-schema-if-needed! :postgres
[driver conn-spec schema]
- (let [sql [[(format "CREATE SCHEMA IF NOT EXISTS \"%s\";" schema)]]]
- (driver/execute-raw-queries! driver conn-spec sql)))
+ ;; Without the blank check, `format` stringifies nil to "null" and creates a schema
+ ;; literally named "null" on the target DB.
+ (when-not (str/blank? schema)
+ (let [sql [[(format "CREATE SCHEMA IF NOT EXISTS \"%s\";" schema)]]]
+ (driver/execute-raw-queries! driver conn-spec sql))))
(defmethod driver/extra-info :postgres
[_driver]
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index 19689a08ac8..212164f5840 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -35,6 +35,7 @@
check-database-feature
check-feature-enabled!
validate-incremental-column-type!
+ validate-target-schema!
validate-transform-query!
get-transforms
get-transform
diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.clj
index de7bf8b22c0..498538aa1c9 100644
--- a/src/metabase/transforms/crud.clj
+++ b/src/metabase/transforms/crud.clj
@@ -3,6 +3,7 @@
so that non-REST modules (e.g. metabot-v3, workspaces) can use them without depending
on the `-rest` module."
(:require
+ [clojure.string :as str]
[metabase.api.common :as api]
[metabase.database-routing.core :as database-routing]
[metabase.driver.util :as driver.u]
@@ -48,6 +49,18 @@
(assoc error
:status-code 400)))))
+(defn validate-target-schema!
+ "Require a non-blank `:target.schema` when the target database supports schemas.
+
+ On schemas-supporting drivers a nil schema makes post-run sync miss the physical table
+ and leaves the Metabase table with zero fields."
+ [transform]
+ (let [db-id (transforms-base.i/target-db-id transform)
+ db (t2/select-one :model/Database db-id)]
+ (when (and db (driver.u/supports? (:engine db) :schemas db))
+ (api/check-400 (not (str/blank? (get-in transform [:target :schema])))
+ (deferred-tru "A target schema is required for this database.")))))
+
(defn validate-incremental-column-type!
"Validates that the checkpoint column for an incremental transform has a supported type.
@@ -100,6 +113,7 @@
([body creator-id]
(when (transforms-base.u/query-transform? body)
(validate-transform-query! body))
+ (validate-target-schema! body)
(let [creator-id (or creator-id api/*current-user-id*)
transform (t2/with-transaction [_]
(let [tag-ids (:tag_ids body)
@@ -134,6 +148,8 @@
;; we must validate on a full transform object
(check-feature-enabled! new)
(check-database-feature new)
+ (when (contains? body :target)
+ (validate-target-schema! new))
(validate-incremental-column-type! new)
(when (transforms-base.u/query-transform? old)
(validate-transform-query! new)
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 2660253a8f9..bcc47b60fdf 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -1094,6 +1094,43 @@
(is (re-find #"CAST" sql))
(is (some? (mt/rows (qp/process-query query)))))))))))
+(deftest create-schema-if-needed-nil-guard-test
+ (testing "create-schema-if-needed! is a no-op when schema is nil or blank (GDGT-2144)"
+ (let [executed-queries (atom [])]
+ (with-redefs [driver/execute-raw-queries! (fn [_driver _conn-spec queries]
+ (swap! executed-queries conj queries))]
+ (driver/create-schema-if-needed! :postgres ::fake-conn nil)
+ (driver/create-schema-if-needed! :postgres ::fake-conn "")
+ (driver/create-schema-if-needed! :postgres ::fake-conn " ")
+ (is (empty? @executed-queries)
+ "nil/blank schema should not issue any SQL")))))
+
+(deftest ^:parallel describe-fields-sql-nil-schema-test
+ (testing "describe-fields-sql for Postgres handles nil schema-names correctly (GDGT-2144)"
+ (let [[nil-schema-sql] (sql-jdbc.sync/describe-fields-sql
+ :postgres
+ {:schema-names [nil]
+ :table-names ["my_table"]
+ :details {}})
+ [mixed-schema-sql] (sql-jdbc.sync/describe-fields-sql
+ :postgres
+ {:schema-names [nil "public"]
+ :table-names ["my_table"]
+ :details {}})
+ [normal-schema-sql] (sql-jdbc.sync/describe-fields-sql
+ :postgres
+ {:schema-names ["public"]
+ :table-names ["my_table"]
+ :details {}})]
+ (is (not (re-find #"(?i)IN \(NULL\)" nil-schema-sql))
+ "rendered SQL must not contain `IN (NULL)` which never matches anything")
+ (is (re-find #"\"table_schema\" IS NULL" nil-schema-sql)
+ "passing [nil] schemas should produce an IS NULL check on table_schema")
+ (is (re-find #"\"table_schema\" IN .+OR .+\"table_schema\" IS NULL" mixed-schema-sql)
+ "mixed nil + non-nil schemas should include both IN and IS NULL")
+ (is (not (re-find #"\"table_schema\" IS NULL" normal-schema-sql))
+ "non-nil-only schemas should not have IS NULL on table_schema"))))
+
;; API tests are in [[metabase.actions-rest.api-test]]
(deftest ^:parallel actions-maybe-parse-sql-violate-not-null-constraint-test
(testing "violate not null constraint"
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index c2b58350d6a..2bd6f5a9e19 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -351,6 +351,18 @@
(is (false? (:is_writable table))
"Computed transform tables should have is_writable=false")))))))
+(deftest target-table-handles-nil-schema-test
+ (testing "target-table can find a Table row with schema=nil (GDGT-2144)"
+ (mt/with-temp [:model/Database {db-id :id} {:engine :h2}
+ :model/Table _table {:db_id db-id
+ :schema nil
+ :name "some_nil_schema_table"
+ :active true}]
+ (let [found (transforms-base.u/target-table db-id {:schema nil :name "some_nil_schema_table"})]
+ (is (some? found)
+ "target-table must find a Table row with schema=nil — otherwise schemas-less drivers (MySQL) would silently miss their targets")
+ (is (= "some_nil_schema_table" (:name found)))))))
+
(deftest execute-sets-transform-id-on-target-table-test
(testing "Executing a query transform sets transform_id on the target table"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index 45ccba08554..a5f1b882223 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -3,6 +3,8 @@
(:require
[clojure.test :refer :all]
[medley.core :as m]
+ [metabase.driver :as driver]
+ [metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.permissions.core :as perms]
@@ -92,6 +94,59 @@
(is (map? (:owner response)))
(is (= lucky-id (get-in response [:owner :id]))))))))))))
+(deftest update-transform-without-schema-test
+ (testing "Updating a transform to clear its schema is rejected on schemas-supporting databases"
+ (mt/with-premium-features #{}
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (when (driver.u/supports? driver/*driver* :schemas (mt/db))
+ (with-transform-cleanup! [table-name "update_schema_products"]
+ (let [query (make-query "Gadget")
+ schema (get-test-schema)
+ created (mt/user-http-request :crowberto :post 200 "transform"
+ {:name "Update Schema Products"
+ :source {:type "query"
+ :query query}
+ :target {:type "table"
+ :schema schema
+ :name table-name}})
+ transform-id (:id created)]
+ (testing "PUT with a nil schema in target is rejected"
+ (mt/user-http-request :crowberto :put 400
+ (format "transform/%s" transform-id)
+ {:target {:type "table"
+ :schema nil
+ :name table-name}}))
+ (testing "PUT with a non-blank schema still succeeds"
+ (let [updated (mt/user-http-request :crowberto :put 200
+ (format "transform/%s" transform-id)
+ {:name "Renamed"})]
+ (is (= "Renamed" (:name updated)))))))))))))
+
+(deftest create-transform-without-schema-test
+ (testing "Creating a transform without a schema is rejected on databases that support schemas, and allowed on those that don't"
+ (mt/with-premium-features #{}
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (mt/with-data-analyst-role! (mt/user->id :lucky)
+ (mt/with-db-perm-for-group! (perms-group/all-users) (mt/id) :perms/transforms :yes
+ (with-transform-cleanup! [table-name "no_schema_products"]
+ (let [query (make-query "Gadget")
+ request (fn [schema]
+ {:name "No Schema Products"
+ :source {:type "query"
+ :query query}
+ :target {:type "table"
+ :schema schema
+ :name table-name}})]
+ (if (driver.u/supports? driver/*driver* :schemas (mt/db))
+ (testing "nil schema is rejected with 400 on schemas-supporting driver"
+ (let [response (mt/user-http-request :lucky :post 400 "transform" (request nil))]
+ (is (nil? (:id response)))))
+ (testing "nil schema is allowed on non-schemas drivers"
+ (let [response (mt/user-http-request :lucky :post 200 "transform" (request nil))]
+ (is (some? (:id response)))))))))))))))
+
(deftest create-transform-with-param-test
(mt/with-premium-features #{}
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)