The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
sql-jdbc: stop munging dashes in schema-qualified table identifiers (#78969)
Timothy S. Dean5a232bb5c891
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex 5306dd6c3bc..10acbab63fe 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -2346,15 +2346,16 @@;; so the CTAS lands in that database. The target namespace lives at the;; AST `:db` position and the canonical `output-table` is bare.;;- ;; HoneySQL renders `(keyword ns name)` as ``ns`.`name`` on MySQL — we- ;; lean on that here. 3-part `:db.:schema.:name` writes (Snowflake / SQL- ;; Server / BigQuery cross-DB) aren't expressible through this single-- ;; keyword shape; supporting those would need `output-table` to carry- ;; both qualifiers or a different HoneySQL form.+ ;; 3-part `db.schema.name` writes (Snowflake / SQL Server / BigQuery+ ;; cross-DB) aren't expressible here; supporting those would need+ ;; `output-table` to carry both qualifiers.target-id (cond(and (not (str/blank? output-db))(str/blank? (namespace (keyword output-table))))- (keyword output-db (clojure.core/name (keyword output-table)))+ ;; a dot-joined string, not (keyword output-db name): HoneySQL munges dashes to+ ;; underscores in a keyword's namespace, so a catalog named test-data would+ ;; target the nonexistent test_data. A string splits on `.`, quoted verbatim.+ (str output-db "." (clojure.core/name (keyword output-table))):else(keyword output-table))][(first (format-honeysql driverdiff --git a/src/metabase/driver/sql_jdbc.clj b/src/metabase/driver/sql_jdbc.cljindex b6dc171a7e5..037ca776d35 100644--- a/src/metabase/driver/sql_jdbc.clj+++ b/src/metabase/driver/sql_jdbc.clj@@ -13,7 +13,7 @@[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql-jdbc.metadata :as sql-jdbc.metadata]- [metabase.driver.sql-jdbc.quoting :refer [quote-columns quote-identifier+ [metabase.driver.sql-jdbc.quoting :refer [dot-qualified quote-columns quote-identifierquote-table with-quoting]][metabase.driver.sql-jdbc.sync :as sql-jdbc.sync][metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]@@ -160,11 +160,14 @@(jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec database-id)](jdbc/execute! conn sql))))+(defn- drop-table-sql [driver table-name]+ (first (sql/format {:drop-table [:if-exists (dot-qualified table-name)]}+ :quoted true+ :dialect (sql.qp/quote-style driver))))+(defmethod driver/drop-table! :sql-jdbc[driver db-id table-name]- (let [sql (first (sql/format {:drop-table [:if-exists (keyword table-name)]}- :quoted true- :dialect (sql.qp/quote-style driver)))]+ (let [sql (drop-table-sql driver table-name)](jdbc/with-db-transaction [conn (sql-jdbc.conn/db->pooled-connection-spec db-id)](jdbc/execute! conn sql))))diff --git a/src/metabase/driver/sql_jdbc/quoting.clj b/src/metabase/driver/sql_jdbc/quoting.cljindex d470fc23219..2156c35b228 100644--- a/src/metabase/driver/sql_jdbc/quoting.clj+++ b/src/metabase/driver/sql_jdbc/quoting.clj@@ -10,10 +10,22 @@sql/*options* (assoc @#'sql/*options* :quoted true)]~@body))+(defn dot-qualified+ "Rewrite a schema-qualified keyword so the whole dotted path lives in its name: `:schema/tbl` -> `:schema.tbl`.+ Takes a keyword or a dot-qualified string; returns a keyword."+ [table-name]+ ;; not (keyword table-name) straight through: HoneySQL munges dashes to underscores in a+ ;; keyword's namespace but not in its name, so a catalog named `test-data` would reach the+ ;; database as `test_data`.+ (let [kw (keyword table-name)]+ (keyword (if-let [schema (namespace kw)]+ (str schema "." (name kw))+ (name kw)))))+(defn quote-table"Protect against a table being interpreted as a function call."[table-name]- (keyword (str "'" (sql/format-entity (keyword table-name)))))+ (keyword (str "'" (sql/format-entity (dot-qualified table-name)))))(defn quote-identifier"Quote an identifier, in case it looks like a function call."diff --git a/test/metabase/driver/sql/transforms_test.clj b/test/metabase/driver/sql/transforms_test.cljindex c91cf4e9f7d..39e49f21874 100644--- a/test/metabase/driver/sql/transforms_test.clj+++ b/test/metabase/driver/sql/transforms_test.clj@@ -4,6 +4,7 @@[clojure.test :refer :all][metabase.driver :as driver][metabase.driver.sql.query-processor :as sql.qp]+ [metabase.driver.sql.util :as sql.u][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.query-processor :as qp]@@ -53,6 +54,26 @@(re-find #"(?i)CREATE\s+TABLE.*AS" sql)(re-find #"(?i)CREATE\s+.*TABLE" sql))))))))))+(deftest ^:parallel compile-transform-output-db-preserves-dashes-test+ (testing "a dashed catalog in :output-db compiles verbatim -- where schema == database the CI+ catalog is named test-data; munged to test_data, the CTAS targets a nonexistent database+ (MySQL error 1049)."+ ;; Only the shared `:sql` implementation reads :output-db. Drivers that override+ ;; compile-transform qualify their target differently and drop it, so they exclude themselves.+ ;; `the-initialized-driver`, not `the-driver`: a driver registers from its plugin manifest with+ ;; its namespace unloaded, and only initialization runs the manifest's `load-namespace` step. Ask+ ;; `get-method` before that and every override still reads as the `:sql` default.+ (doseq [driver (map driver/the-initialized-driver (descendants driver/hierarchy :sql-jdbc))+ :when (identical? (get-method driver/compile-transform driver)+ (get-method driver/compile-transform :sql))]+ (testing driver+ (let [[sql] (driver/compile-transform driver {:query {:query "SELECT 1"}+ :output-db "test-data"+ :output-table :some_out_tbl})]+ (is (re-find #"test-data" sql) "dashed catalog must be quoted verbatim")+ (is (not (re-find #"test_data" sql)) "the dash must not be munged to an underscore")+ (is (re-find #"some_out_tbl" sql) "the bare output-table segment is still present"))))))+(deftest compile-drop-table-contract-test(testing "compile-drop-table should return [sql params] format"(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)@@ -274,3 +295,29 @@(is (= written (:rows-affected insert-result))(format "%s: INSERT :rows-affected (%s) should equal %s — failure means the driver also undercounts DML"driver/*driver* (pr-str (:rows-affected insert-result)) written)))))))))++(deftest transform-target-in-dashed-schema-round-trips-test+ ;; H2 only -- no `drop-schema!` multimethod exists, so a wider sweep would litter shared CI accounts.+ (mt/test-driver :h2+ (mt/with-premium-features #{:transforms-basic}+ (let [dashed-schema "dashed-schema-probe"+ conn-spec (driver/connection-spec driver/*driver* (mt/db))+ quoted-schema (sql.u/quote-name driver/*driver* :schema dashed-schema)+ execute! #(driver/execute-raw-queries! driver/*driver* conn-spec [[%]])]+ (execute! (str "CREATE SCHEMA IF NOT EXISTS " quoted-schema))+ (try+ (transforms.tu/with-transform-cleanup! [target {:type :table+ :schema dashed-schema+ :name "dashed_table_probe"+ :database (mt/id)}]+ (let [target-table (:name target)+ exists? #(driver/table-exists? driver/*driver* (mt/db)+ {:schema dashed-schema, :name target-table})]+ (driver/create-table! driver/*driver* (mt/id) (keyword dashed-schema target-table)+ {"id" (driver/type->database-type driver/*driver* :type/Integer)} {})+ (is (exists?) "the target table is created in the dashed schema")+ (transforms.tu/drop-target! target)+ (is (not (exists?)) "and dropping the transform target actually drops the table")))+ (finally+ ;; with-transform-cleanup! has dropped the table by now, so the schema is empty+ (execute! (str "DROP SCHEMA IF EXISTS " quoted-schema))))))))diff --git a/test/metabase/driver/sql_jdbc_test.clj b/test/metabase/driver/sql_jdbc_test.cljindex 066deefbbfd..587354348b9 100644--- a/test/metabase/driver/sql_jdbc_test.clj+++ b/test/metabase/driver/sql_jdbc_test.clj@@ -6,6 +6,7 @@[metabase.driver :as driver][metabase.driver.common.table-rows-sample :as table-rows-sample][metabase.driver.sql-jdbc :as driver.sql-jdbc]+ [metabase.driver.sql-jdbc.quoting :as quoting][metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.util :as driver.u]@@ -393,11 +394,66 @@(when (driver/table-exists? driver (mt/db) {:name test-table :schema schema})(driver/drop-table! driver db-id qualified-table))))))))+(defn- sql-jdbc-drivers+ "Every registered sql-jdbc driver. These tests build SQL without connecting, so they run against the+ whole hierarchy rather than whichever drivers happen to be available."+ []+ (descendants driver/hierarchy :sql-jdbc))+(deftest ^:parallel insert-into-sqls-boolean-literal-test(testing "boolean row values bind as parameters, never as inlined literals -- not everydialect has a boolean literal keyword"- (doseq [driver [:h2]]- (let [[sql & params] (first (#'driver.sql-jdbc/insert-into!-sqls driver :dbo/t ["id" "flag"]- [[1 true] [2 false]] false))]- (is (not (re-find #"(?i)\bTRUE\b|\bFALSE\b" sql)) (str driver))- (is (= [1 true 2 false] params) (str driver))))))+ (doseq [driver (sql-jdbc-drivers)]+ (testing driver+ (let [[sql & params] (first (#'driver.sql-jdbc/insert-into!-sqls driver :dbo/t ["id" "flag"]+ [[1 true] [2 false]] false))]+ (is (not (re-find #"(?i)\bTRUE\b|\bFALSE\b" sql)))+ (is (= [1 true 2 false] params)))))))++(deftest ^:parallel dot-qualified-test+ (testing "the whole dotted path lands in the keyword's name, which HoneySQL leaves alone"+ (are [table-name expected] (= expected (quoting/dot-qualified table-name))+ (keyword "test-data" "some_tbl") :test-data.some_tbl+ (keyword "test-data" "a.b") :test-data.a.b+ (keyword "some_tbl") :some_tbl+ "test-data.tbl" :test-data.tbl+ "some_tbl" :some_tbl)))++(deftest ^:parallel create-table-sql-preserves-dashes-test+ (let [create-sql #(#'driver.sql-jdbc/create-table!-sql %1 %2 [["id" [:int]]])]+ (testing "a dash in a schema/catalog segment survives -- munged to an underscore, CREATE TABLE+ targets a schema that isn't there"+ (doseq [driver (sql-jdbc-drivers)]+ (testing driver+ (let [sql (create-sql driver (keyword "test-data" "some_tbl"))]+ (is (re-find #"test-data" sql))+ (is (not (re-find #"test_data" sql)))))))+ (testing "the whole statement, for one dialect"+ (is (= "CREATE TABLE \"test-data\".\"some_tbl\" (\"id\" INT)"+ (create-sql :h2 (keyword "test-data" "some_tbl")))))+ (testing "unqualified name -- the schema travels in the connection's catalog"+ (is (= "CREATE TABLE \"some_tbl\" (\"id\" INT)"+ (create-sql :h2 (keyword "some_tbl")))))+ (testing "dot-qualified strings split into segments"+ (is (= "CREATE TABLE \"schema\".\"name\" (\"id\" INT)"+ (create-sql :h2 "schema.name"))))))++(deftest ^:parallel drop-table-sql-preserves-dashes-test+ (let [drop-sql #'driver.sql-jdbc/drop-table-sql]+ (testing "a dash in a schema/catalog segment survives -- munged to an underscore, DROP TABLE IF+ EXISTS targets a nonexistent object and silently no-ops, leaking the table"+ (doseq [driver (sql-jdbc-drivers)]+ (testing driver+ (let [sql (drop-sql driver (keyword "test-data" "some_tbl"))]+ (is (re-find #"test-data" sql))+ (is (not (re-find #"test_data" sql)))))))+ (testing "the whole statement, for one dialect"+ (is (= "DROP TABLE IF EXISTS \"test-data\".\"some_tbl\""+ (drop-sql :h2 (keyword "test-data" "some_tbl")))))+ (testing "unqualified name -- the schema travels in the connection's catalog"+ (is (= "DROP TABLE IF EXISTS \"some_tbl\"" (drop-sql :h2 (keyword "some_tbl")))))+ (testing "dot-qualified strings (metabase.upload.impl/table-identifier's shape) split into segments"+ (is (= "DROP TABLE IF EXISTS \"schema\".\"name\"" (drop-sql :h2 "schema.name"))))+ (testing "a dot inside the name splits too -- no call site produces this shape, but keep it uniform"+ (is (= "DROP TABLE IF EXISTS \"test-data\".\"a\".\"b\""+ (drop-sql :h2 (keyword "test-data" "a.b")))))))