Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

sql-jdbc: stop munging dashes in schema-qualified table identifiers (#78969)

Timothy S. Dean5a232bb5c891
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 5306dd6c3bc..10acbab63fe 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -2346,15 +2346,16 @@
;; so the CTAS lands in that database. The target namespace lives at the
;; AST `:db` position and the canonical `output-table` is bare.
;;
- ;; HoneySQL renders `(keyword ns name)` as ``ns`.`name`` on MySQL — we
- ;; lean on that here. 3-part `:db.:schema.:name` writes (Snowflake / SQL
- ;; Server / BigQuery cross-DB) aren't expressible through this single-
- ;; keyword shape; supporting those would need `output-table` to carry
- ;; both qualifiers or a different HoneySQL form.
+ ;; 3-part `db.schema.name` writes (Snowflake / SQL Server / BigQuery
+ ;; cross-DB) aren't expressible here; supporting those would need
+ ;; `output-table` to carry both qualifiers.
target-id (cond
(and (not (str/blank? output-db))
(str/blank? (namespace (keyword output-table))))
- (keyword output-db (clojure.core/name (keyword output-table)))
+ ;; a dot-joined string, not (keyword output-db name): HoneySQL munges dashes to
+ ;; underscores in a keyword's namespace, so a catalog named test-data would
+ ;; target the nonexistent test_data. A string splits on `.`, quoted verbatim.
+ (str output-db "." (clojure.core/name (keyword output-table)))
:else
(keyword output-table))]
[(first (format-honeysql driver
diff --git a/src/metabase/driver/sql_jdbc.clj b/src/metabase/driver/sql_jdbc.clj
index b6dc171a7e5..037ca776d35 100644
--- a/src/metabase/driver/sql_jdbc.clj
+++ b/src/metabase/driver/sql_jdbc.clj
@@ -13,7 +13,7 @@
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.metadata :as sql-jdbc.metadata]
- [metabase.driver.sql-jdbc.quoting :refer [quote-columns quote-identifier
+ [metabase.driver.sql-jdbc.quoting :refer [dot-qualified quote-columns quote-identifier
quote-table with-quoting]]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
@@ -160,11 +160,14 @@
(jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec database-id)]
(jdbc/execute! conn sql))))
+(defn- drop-table-sql [driver table-name]
+ (first (sql/format {:drop-table [:if-exists (dot-qualified table-name)]}
+ :quoted true
+ :dialect (sql.qp/quote-style driver))))
+
(defmethod driver/drop-table! :sql-jdbc
[driver db-id table-name]
- (let [sql (first (sql/format {:drop-table [:if-exists (keyword table-name)]}
- :quoted true
- :dialect (sql.qp/quote-style driver)))]
+ (let [sql (drop-table-sql driver table-name)]
(jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)]
(jdbc/execute! conn sql))))
diff --git a/src/metabase/driver/sql_jdbc/quoting.clj b/src/metabase/driver/sql_jdbc/quoting.clj
index d470fc23219..2156c35b228 100644
--- a/src/metabase/driver/sql_jdbc/quoting.clj
+++ b/src/metabase/driver/sql_jdbc/quoting.clj
@@ -10,10 +10,22 @@
sql/*options* (assoc @#'sql/*options* :quoted true)]
~@body))
+(defn dot-qualified
+ "Rewrite a schema-qualified keyword so the whole dotted path lives in its name: `:schema/tbl` -> `:schema.tbl`.
+ Takes a keyword or a dot-qualified string; returns a keyword."
+ [table-name]
+ ;; not (keyword table-name) straight through: HoneySQL munges dashes to underscores in a
+ ;; keyword's namespace but not in its name, so a catalog named `test-data` would reach the
+ ;; database as `test_data`.
+ (let [kw (keyword table-name)]
+ (keyword (if-let [schema (namespace kw)]
+ (str schema "." (name kw))
+ (name kw)))))
+
(defn quote-table
"Protect against a table being interpreted as a function call."
[table-name]
- (keyword (str "'" (sql/format-entity (keyword table-name)))))
+ (keyword (str "'" (sql/format-entity (dot-qualified table-name)))))
(defn quote-identifier
"Quote an identifier, in case it looks like a function call."
diff --git a/test/metabase/driver/sql/transforms_test.clj b/test/metabase/driver/sql/transforms_test.clj
index c91cf4e9f7d..39e49f21874 100644
--- a/test/metabase/driver/sql/transforms_test.clj
+++ b/test/metabase/driver/sql/transforms_test.clj
@@ -4,6 +4,7 @@
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.driver.sql.util :as sql.u]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor :as qp]
@@ -53,6 +54,26 @@
(re-find #"(?i)CREATE\s+TABLE.*AS" sql)
(re-find #"(?i)CREATE\s+.*TABLE" sql))))))))))
+(deftest ^:parallel compile-transform-output-db-preserves-dashes-test
+ (testing "a dashed catalog in :output-db compiles verbatim -- where schema == database the CI
+ catalog is named test-data; munged to test_data, the CTAS targets a nonexistent database
+ (MySQL error 1049)."
+ ;; Only the shared `:sql` implementation reads :output-db. Drivers that override
+ ;; compile-transform qualify their target differently and drop it, so they exclude themselves.
+ ;; `the-initialized-driver`, not `the-driver`: a driver registers from its plugin manifest with
+ ;; its namespace unloaded, and only initialization runs the manifest's `load-namespace` step. Ask
+ ;; `get-method` before that and every override still reads as the `:sql` default.
+ (doseq [driver (map driver/the-initialized-driver (descendants driver/hierarchy :sql-jdbc))
+ :when (identical? (get-method driver/compile-transform driver)
+ (get-method driver/compile-transform :sql))]
+ (testing driver
+ (let [[sql] (driver/compile-transform driver {:query {:query "SELECT 1"}
+ :output-db "test-data"
+ :output-table :some_out_tbl})]
+ (is (re-find #"test-data" sql) "dashed catalog must be quoted verbatim")
+ (is (not (re-find #"test_data" sql)) "the dash must not be munged to an underscore")
+ (is (re-find #"some_out_tbl" sql) "the bare output-table segment is still present"))))))
+
(deftest compile-drop-table-contract-test
(testing "compile-drop-table should return [sql params] format"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
@@ -274,3 +295,29 @@
(is (= written (:rows-affected insert-result))
(format "%s: INSERT :rows-affected (%s) should equal %s — failure means the driver also undercounts DML"
driver/*driver* (pr-str (:rows-affected insert-result)) written)))))))))
+
+(deftest transform-target-in-dashed-schema-round-trips-test
+ ;; H2 only -- no `drop-schema!` multimethod exists, so a wider sweep would litter shared CI accounts.
+ (mt/test-driver :h2
+ (mt/with-premium-features #{:transforms-basic}
+ (let [dashed-schema "dashed-schema-probe"
+ conn-spec (driver/connection-spec driver/*driver* (mt/db))
+ quoted-schema (sql.u/quote-name driver/*driver* :schema dashed-schema)
+ execute! #(driver/execute-raw-queries! driver/*driver* conn-spec [[%]])]
+ (execute! (str "CREATE SCHEMA IF NOT EXISTS " quoted-schema))
+ (try
+ (transforms.tu/with-transform-cleanup! [target {:type :table
+ :schema dashed-schema
+ :name "dashed_table_probe"
+ :database (mt/id)}]
+ (let [target-table (:name target)
+ exists? #(driver/table-exists? driver/*driver* (mt/db)
+ {:schema dashed-schema, :name target-table})]
+ (driver/create-table! driver/*driver* (mt/id) (keyword dashed-schema target-table)
+ {"id" (driver/type->database-type driver/*driver* :type/Integer)} {})
+ (is (exists?) "the target table is created in the dashed schema")
+ (transforms.tu/drop-target! target)
+ (is (not (exists?)) "and dropping the transform target actually drops the table")))
+ (finally
+ ;; with-transform-cleanup! has dropped the table by now, so the schema is empty
+ (execute! (str "DROP SCHEMA IF EXISTS " quoted-schema))))))))
diff --git a/test/metabase/driver/sql_jdbc_test.clj b/test/metabase/driver/sql_jdbc_test.clj
index 066deefbbfd..587354348b9 100644
--- a/test/metabase/driver/sql_jdbc_test.clj
+++ b/test/metabase/driver/sql_jdbc_test.clj
@@ -6,6 +6,7 @@
[metabase.driver :as driver]
[metabase.driver.common.table-rows-sample :as table-rows-sample]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
+ [metabase.driver.sql-jdbc.quoting :as quoting]
[metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.util :as driver.u]
@@ -393,11 +394,66 @@
(when (driver/table-exists? driver (mt/db) {:name test-table :schema schema})
(driver/drop-table! driver db-id qualified-table))))))))
+(defn- sql-jdbc-drivers
+ "Every registered sql-jdbc driver. These tests build SQL without connecting, so they run against the
+ whole hierarchy rather than whichever drivers happen to be available."
+ []
+ (descendants driver/hierarchy :sql-jdbc))
+
(deftest ^:parallel insert-into-sqls-boolean-literal-test
(testing "boolean row values bind as parameters, never as inlined literals -- not every
dialect has a boolean literal keyword"
- (doseq [driver [:h2]]
- (let [[sql & params] (first (#'driver.sql-jdbc/insert-into!-sqls driver :dbo/t ["id" "flag"]
- [[1 true] [2 false]] false))]
- (is (not (re-find #"(?i)\bTRUE\b|\bFALSE\b" sql)) (str driver))
- (is (= [1 true 2 false] params) (str driver))))))
+ (doseq [driver (sql-jdbc-drivers)]
+ (testing driver
+ (let [[sql & params] (first (#'driver.sql-jdbc/insert-into!-sqls driver :dbo/t ["id" "flag"]
+ [[1 true] [2 false]] false))]
+ (is (not (re-find #"(?i)\bTRUE\b|\bFALSE\b" sql)))
+ (is (= [1 true 2 false] params)))))))
+
+(deftest ^:parallel dot-qualified-test
+ (testing "the whole dotted path lands in the keyword's name, which HoneySQL leaves alone"
+ (are [table-name expected] (= expected (quoting/dot-qualified table-name))
+ (keyword "test-data" "some_tbl") :test-data.some_tbl
+ (keyword "test-data" "a.b") :test-data.a.b
+ (keyword "some_tbl") :some_tbl
+ "test-data.tbl" :test-data.tbl
+ "some_tbl" :some_tbl)))
+
+(deftest ^:parallel create-table-sql-preserves-dashes-test
+ (let [create-sql #(#'driver.sql-jdbc/create-table!-sql %1 %2 [["id" [:int]]])]
+ (testing "a dash in a schema/catalog segment survives -- munged to an underscore, CREATE TABLE
+ targets a schema that isn't there"
+ (doseq [driver (sql-jdbc-drivers)]
+ (testing driver
+ (let [sql (create-sql driver (keyword "test-data" "some_tbl"))]
+ (is (re-find #"test-data" sql))
+ (is (not (re-find #"test_data" sql)))))))
+ (testing "the whole statement, for one dialect"
+ (is (= "CREATE TABLE \"test-data\".\"some_tbl\" (\"id\" INT)"
+ (create-sql :h2 (keyword "test-data" "some_tbl")))))
+ (testing "unqualified name -- the schema travels in the connection's catalog"
+ (is (= "CREATE TABLE \"some_tbl\" (\"id\" INT)"
+ (create-sql :h2 (keyword "some_tbl")))))
+ (testing "dot-qualified strings split into segments"
+ (is (= "CREATE TABLE \"schema\".\"name\" (\"id\" INT)"
+ (create-sql :h2 "schema.name"))))))
+
+(deftest ^:parallel drop-table-sql-preserves-dashes-test
+ (let [drop-sql #'driver.sql-jdbc/drop-table-sql]
+ (testing "a dash in a schema/catalog segment survives -- munged to an underscore, DROP TABLE IF
+ EXISTS targets a nonexistent object and silently no-ops, leaking the table"
+ (doseq [driver (sql-jdbc-drivers)]
+ (testing driver
+ (let [sql (drop-sql driver (keyword "test-data" "some_tbl"))]
+ (is (re-find #"test-data" sql))
+ (is (not (re-find #"test_data" sql)))))))
+ (testing "the whole statement, for one dialect"
+ (is (= "DROP TABLE IF EXISTS \"test-data\".\"some_tbl\""
+ (drop-sql :h2 (keyword "test-data" "some_tbl")))))
+ (testing "unqualified name -- the schema travels in the connection's catalog"
+ (is (= "DROP TABLE IF EXISTS \"some_tbl\"" (drop-sql :h2 (keyword "some_tbl")))))
+ (testing "dot-qualified strings (metabase.upload.impl/table-identifier's shape) split into segments"
+ (is (= "DROP TABLE IF EXISTS \"schema\".\"name\"" (drop-sql :h2 "schema.name"))))
+ (testing "a dot inside the name splits too -- no call site produces this shape, but keep it uniform"
+ (is (= "DROP TABLE IF EXISTS \"test-data\".\"a\".\"b\""
+ (drop-sql :h2 (keyword "test-data" "a.b")))))))