Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Upgrade clickhouse to 0.9.5 (#67297)

Riley Thompson88ec74f7b35b
diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
index 775b2b71b2e..54cd1715c5b 100644
--- a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
@@ -803,3 +803,31 @@
(mt/first-row
(mt/run-mbql-query venues
{:aggregation [[:count]]}))))))))))))))))
+
+(deftest clickhouse-double-hyphen-test
+ (testing "can use impersonation on clickhouse with role containing a double hyphen (#57016)"
+ (mt/test-driver :clickhouse
+ (mt/with-premium-features #{:advanced-permissions}
+ (let [venues-table (sql.tx/qualify-and-quote driver/*driver* "test-data" "venues")
+ role-a "role--with--double--hyphens"]
+ (tx/with-temp-roles! driver/*driver*
+ (impersonation-granting-details driver/*driver* (mt/db))
+ {role-a {venues-table {}}}
+ (impersonation-default-user driver/*driver*)
+ (impersonation-default-role driver/*driver*)
+ (mt/with-temp [:model/Database database {:engine driver/*driver*,
+ :details (impersonation-details driver/*driver* (mt/db))}]
+ (mt/with-db database
+ (when (driver/database-supports? driver/*driver* :connection-impersonation-requires-role nil)
+ (t2/update! :model/Database :id (mt/id) (assoc-in (mt/db) [:details :role] (impersonation-default-role driver/*driver*))))
+ (sync/sync-database! database {:scan :schema})
+ (impersonation.util-test/with-impersonations! {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]
+ :attributes {"impersonation_attr" role-a}}
+ (is (= [[100]]
+ (mt/formatted-rows [int]
+ (mt/run-mbql-query venues
+ {:aggregation [[:count]]}))))
+ (is (thrown?
+ java.lang.Exception
+ (mt/run-mbql-query checkins
+ {:aggregation [[:count]]}))))))))))))
diff --git a/modules/drivers/clickhouse/deps.edn b/modules/drivers/clickhouse/deps.edn
index 8d691785528..7930df7336a 100644
--- a/modules/drivers/clickhouse/deps.edn
+++ b/modules/drivers/clickhouse/deps.edn
@@ -1,5 +1,5 @@
{:paths ["src" "resources"]
:deps
- {com.clickhouse/clickhouse-jdbc {:mvn/version "0.8.4"
+ {com.clickhouse/clickhouse-jdbc {:mvn/version "0.9.5"
:exclusions [org.apache.commons/commons-lang3]}
org.lz4/lz4-java {:mvn/version "1.8.0"}}}
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
index eb1716afef5..62df8965243 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
@@ -20,7 +20,8 @@
[metabase.util :as u]
[metabase.util.log :as log])
(:import [com.clickhouse.client.api.query QuerySettings]
- [java.sql SQLException]))
+ [java.sql SQLException PreparedStatement]
+ [java.time LocalDate]))
(set! *warn-on-reflection* true)
@@ -347,3 +348,9 @@
[_driver _database _table]
(log/warn "Clickhouse does not support foreign keys. `describe-table-fks` should not have been called!")
#{})
+
+;; Override clickhouse to not pass in the Types/DATE parameter due to jdbc
+;; driver issue: https://github.com/ClickHouse/clickhouse-java/issues/2701
+(defmethod sql-jdbc.execute/set-parameter [:clickhouse LocalDate]
+ [_ ^PreparedStatement prepared-statement i object]
+ (.setObject prepared-statement i object))
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj
index db560c6ff1c..77632a486c5 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_data_types_test.clj
@@ -441,19 +441,19 @@
[{:field-name "mystring" :base-type :type/Text}]
[["foo"] ["bar"] [" "] [""] [nil]]]])
(testing "null strings count"
- (is (= 2M
+ (is (= 2
(-> (mt/run-mbql-query test-data-nullable-strings
{:filter [:is-null $mystring]
:aggregation [:count]})
mt/first-row last))))
(testing "nullable strings not null filter"
- (is (= 3M
+ (is (= 3
(-> (mt/run-mbql-query test-data-nullable-strings
{:filter [:not-null $mystring]
:aggregation [:count]})
mt/first-row last))))
(testing "filter nullable string by value"
- (is (= 1M
+ (is (= 1
(-> (mt/run-mbql-query test-data-nullable-strings
{:filter [:= $mystring "foo"]
:aggregation [:count]})
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj
index 47909658c67..ad9a9544326 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_introspection_test.clj
@@ -1,5 +1,4 @@
(ns ^:mb/driver-tests metabase.driver.clickhouse-introspection-test
- #_{:clj-kondo/ignore [:unsorted-required-namespaces]}
(:require
[clojure.test :refer :all]
[metabase.driver :as driver]
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_substitution_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_substitution_test.clj
index 818a43ecfd7..413c52d3339 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_substitution_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_substitution_test.clj
@@ -338,8 +338,8 @@
(deftest clickhouse-native-query-with-uuid-filter-test
(mt/test-driver :clickhouse
- (let [uuid-1 (random-uuid)
- uuid-2 (random-uuid)]
+ (let [uuid-1 #uuid "3127abff-e634-4114-a015-59893b49ae74"
+ uuid-2 #uuid "a65e1a3b-4710-4136-b5b7-63a747e17a5a"]
(mt/dataset
(mt/dataset-definition "uuid_filter_db"
[["uuid_filter_table"
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
index 0e442dbb75c..ff4d554c68f 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
@@ -6,6 +6,7 @@
[metabase.driver.clickhouse-qp :as clickhouse-qp]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+ [metabase.lib.card :as lib.card]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor :as qp]
@@ -208,7 +209,7 @@
(deftest ^:parallel comment-question-mark-test
(mt/test-driver :clickhouse
- (testing "a query with a question mark in the comment and has a variable should work correctly"
+ (testing "a query with a question mark in the comment and has a variable should work correctly (#56690)"
(let [query "SELECT *
-- ?
FROM test_data.categories
@@ -228,7 +229,7 @@
(deftest ^:parallel select-question-mark-test
(mt/test-driver :clickhouse
- (testing "a query that selects a question mark and has a variable should work correctly"
+ (testing "a query that selects a question mark and has a variable should work correctly (#56690)"
(let [query "SELECT *, '?'
FROM test_data.categories
WHERE {{category_name}};"]
@@ -249,36 +250,34 @@
:target [:dimension [:template-tag "category_name"]]
:value ["African"]}]}))))))))
-;; TODO(rileythomp, 2025-09-23): Enable when ClickHouse JDBC driver has been fixed
-#_(deftest ^:parallel ternary-with-variable-test
- (mt/test-driver :clickhouse
- (testing "a query with a ternary and a variable should work correctly"
- (is (= [[1 "African" 1]]
- (mt/rows
- (qp/process-query
- {:database (mt/id)
- :type :native
- :native {:query "SELECT *, true ? 1 : 0 AS foo
- FROM test_data.categories
- WHERE name = {{category_name}};"
- :template-tags {"category_name" {:type :text
- :name "category_name"
- :display-name "Category Name"}}}
- :parameters [{:type :category
- :target [:variable [:template-tag "category_name"]]
- :value "African"}]})))))))
+(deftest ^:parallel ternary-with-variable-test
+ (mt/test-driver :clickhouse
+ (testing "a query with a ternary and a variable should work correctly (#56690)"
+ (is (= [[1 "African" 1]]
+ (mt/rows
+ (qp/process-query
+ {:database (mt/id)
+ :type :native
+ :native {:query "SELECT *, true ? 1 : 0 AS foo
+ FROM test_data.categories
+ WHERE name = {{category_name}};"
+ :template-tags {"category_name" {:type :text
+ :name "category_name"
+ :display-name "Category Name"}}}
+ :parameters [{:type :category
+ :target [:variable [:template-tag "category_name"]]
+ :value "African"}]})))))))
-;; TODO(rileythomp, 2025-09-23): Enable when ClickHouse JDBC driver has been fixed
-#_(deftest ^:parallel line-comment-block-comment-test
- (mt/test-driver :clickhouse
- (testing "a query with a line comment followed by a block comment should work correctly"
- (is (= [[1]]
- (mt/rows
- (qp/process-query
- (mt/native-query
- {:query "-- foo
- /* comment */
- select 1;"}))))))))
+(deftest ^:parallel line-comment-block-comment-test
+ (mt/test-driver :clickhouse
+ (testing "a query with a line comment followed by a block comment should work correctly (#57149, #62741)"
+ (is (= [[1]]
+ (mt/rows
+ (qp/process-query
+ (mt/native-query
+ {:query "-- foo
+ /* comment */
+ select 1;"}))))))))
(deftest ^:parallel subquery-with-cte-test
(mt/test-driver :clickhouse
@@ -372,3 +371,20 @@
:parameters [{:type "string/="
:target [:variable [:template-tag "val"]]
:value ["abc"]}]})))))))
+
+(deftest ^:parallel native-query-cte-filtering-test
+ (mt/test-driver :clickhouse
+ (testing "can filter on a saved native query with a CTE (#63635)"
+ (let [native-query (mt/native-query
+ {:query "with base as (select 1 id, 'abc' val) select * from base"})
+ card-data (mt/card-with-source-metadata-for-query native-query)]
+ (mt/with-temp [:model/Card {card-id :id} card-data]
+ (let [mp (mt/metadata-provider)
+ card-mp (lib.metadata/card mp card-id)
+ val-col (some #(when (= "val" (:name %)) %)
+ (lib.card/card-returned-columns mp card-mp))]
+ (is (= [[1 "abc"]]
+ (-> (lib/query mp card-mp)
+ (lib/filter (lib/= val-col "abc"))
+ (qp/process-query)
+ (mt/rows))))))))))
diff --git a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
index abafd001b84..322564b97ae 100644
--- a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
+++ b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
@@ -239,11 +239,25 @@
;; if the ResultSet returns anything we know the table is already loaded.
(.next rset))))))
-(defn grant-table-perms-to-roles!
+(defn- quote-role
+ "Similar to sql.tx/qualify-and-quote but avoid replacing - with _"
+ [role]
+ (->> (apply sql.tx/qualified-name-components :clickhouse [role])
+ (apply sql.u/quote-name :clickhouse :field)))
+
+(defn- drop-if-exists-and-create-roles! [driver details roles]
+ (let [spec (sql-jdbc.conn/connection-details->spec driver details)]
+ (doseq [[role-name _table-perms] roles]
+ (let [role-name (quote-role role-name)]
+ (doseq [statement [(format "DROP ROLE IF EXISTS %s;" role-name)
+ (format "CREATE ROLE %s;" role-name)]]
+ (jdbc/execute! spec [statement] {:transaction? false}))))))
+
+(defn- grant-table-perms-to-roles!
[driver details roles]
(let [spec (sql-jdbc.conn/connection-details->spec driver details)]
(doseq [[role-name table-perms] roles]
- (let [role-name (sql.tx/qualify-and-quote driver role-name)]
+ (let [role-name (quote-role role-name)]
(doseq [[table-name perms] table-perms]
(jdbc/execute! spec [(format "GRANT SELECT ON %s TO %s" table-name role-name)] {:transaction? false})
(when-let [rls-perms (:rls perms)]
@@ -252,6 +266,15 @@
(jdbc/execute! spec [(format "CREATE ROW POLICY role_policy_%s ON %s AS RESTRICTIVE FOR SELECT USING %s TO %s"
(mt/random-name) table-name policy-cond role-name)] {:transaction? false}))))))))
+(defn- grant-roles-to-user! [driver details roles user-name]
+ (let [spec (sql-jdbc.conn/connection-details->spec driver details)
+ user-name (sql.tx/qualify-and-quote driver user-name)]
+ (doseq [[role-name _table-perms] roles]
+ (let [role-name (quote-role role-name)]
+ (jdbc/execute! spec
+ [(format "GRANT %s TO %s" role-name user-name)]
+ {:transaction? false})))))
+
(defmethod tx/create-and-grant-roles! :clickhouse
[driver details roles user-name _default-role]
(let [spec (sql-jdbc.conn/connection-details->spec driver details)]
@@ -259,9 +282,9 @@
(doseq [statement [(format "DROP USER IF EXISTS %s" user-name)
(format "CREATE USER IF NOT EXISTS %s NOT IDENTIFIED;" user-name)]]
(jdbc/execute! spec [statement] {:transaction? false})))
- (sql-jdbc.tx/drop-if-exists-and-create-roles! driver details roles)
+ (drop-if-exists-and-create-roles! driver details roles)
(grant-table-perms-to-roles! driver details roles)
- (sql-jdbc.tx/grant-roles-to-user! driver details roles user-name))
+ (grant-roles-to-user! driver details roles user-name))
(defmethod tx/drop-roles! :clickhouse
[driver details roles user-name]