Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix ssh tunneling for clickhouse (#63464)

Eric Normand6afcf2aed582
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
index dff992030f4..f82df5d5ab4 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
@@ -1,7 +1,6 @@
(ns metabase.driver.clickhouse
"Driver for ClickHouse databases"
(:require
- [clojure.core.memoize :as memoize]
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
[metabase.driver :as driver]
@@ -68,33 +67,6 @@
(def ^:private default-connection-details
{:user "default" :password "" :dbname "default" :host "localhost" :port 8123})
-(defn- connection-details->spec* [details]
- (let [;; ensure defaults merge on top of nils
- details (reduce-kv (fn [m k v] (assoc m k (or v (k default-connection-details))))
- default-connection-details
- details)
- {:keys [user password dbname host port ssl clickhouse-settings max-open-connections]} details
- host (cond ; JDBCv1 used to accept schema in the `host` configuration option
- (str/starts-with? host "http://") (subs host 7)
- (str/starts-with? host "https://") (subs host 8)
- :else host)]
- (-> {:classname "com.clickhouse.jdbc.ClickHouseDriver"
- :subprotocol "clickhouse"
- :subname (str "//" host ":" port "/" dbname)
- :password (or password "")
- :user user
- :ssl (boolean ssl)
- :use_server_time_zone_for_dates true
- :product_name (format "metabase/%s" (:tag driver-api/mb-version-info))
- :remember_last_set_roles true
- :http_connection_provider "HTTP_URL_CONNECTION"
- :jdbc_ignore_unsupported_values "true"
- :jdbc_schema_term "schema"
- :max_open_connections (or max-open-connections 100)
- ;; see also: https://clickhouse.com/docs/en/integrations/java#configuration
- :custom_http_params (or clickhouse-settings "")}
- (sql-jdbc.common/handle-additional-options details :separator-style :url))))
-
(defmethod sql-jdbc.execute/do-with-connection-with-options :clickhouse
[driver db-or-id-or-spec {:keys [^String session-timezone _write?] :as options} f]
(sql-jdbc.execute/do-with-resolved-connection
@@ -122,37 +94,37 @@
(sql-jdbc.execute/set-time-zone-if-supported! driver conn session-timezone))
(f conn))))
-(def ^:private ^{:arglists '([db-details])} cloud?
- "Returns true if the `db-details` are for a ClickHouse Cloud instance, and false otherwise. If it fails to connect
- to the database, it throws a java.sql.SQLException."
- (memoize/ttl
- (fn [db-details]
- (let [spec (connection-details->spec* db-details)]
- (sql-jdbc.execute/do-with-connection-with-options
- :clickhouse spec nil
- (fn [^java.sql.Connection conn]
- (with-open [stmt (.createStatement conn)
- rset (.executeQuery stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")]
- (if (.next rset) (.getBoolean rset 1) false))))))
- ;; cache the results for 48 hours; TTL is here only to eventually clear out old entries
- :ttl/threshold (* 48 60 60 1000)))
-
(defmethod sql-jdbc.conn/connection-details->spec :clickhouse
[_ details]
- (cond-> (connection-details->spec* details)
- (try (cloud? details)
- (catch java.sql.SQLException _e
- false))
- ;; select_sequential_consistency guarantees that we can query data from any replica in CH Cloud
- ;; immediately after it is written
- (assoc :select_sequential_consistency true)))
+ (let [;; ensure defaults merge on top of nils
+ details (reduce-kv (fn [m k v] (assoc m k (or v (k default-connection-details))))
+ default-connection-details
+ details)
+ {:keys [user password dbname host port ssl clickhouse-settings max-open-connections]} details
+ host (cond ; JDBCv1 used to accept schema in the `host` configuration option
+ (str/starts-with? host "http://") (subs host 7)
+ (str/starts-with? host "https://") (subs host 8)
+ :else host)]
+ (-> {:classname "com.clickhouse.jdbc.ClickHouseDriver"
+ :subprotocol "clickhouse"
+ :subname (str "//" host ":" port "/" dbname)
+ :password (or password "")
+ :user user
+ :ssl (boolean ssl)
+ :use_server_time_zone_for_dates true
+ :product_name (format "metabase/%s" (:tag driver-api/mb-version-info))
+ :remember_last_set_roles true
+ :http_connection_provider "HTTP_URL_CONNECTION"
+ :jdbc_ignore_unsupported_values "true"
+ :jdbc_schema_term "schema"
+ :select_sequential_consistency true
+ :max_open_connections (or max-open-connections 100)
+ ;; see also: https://clickhouse.com/docs/en/integrations/java#configuration
+ :custom_http_params (or clickhouse-settings "")}
+ (sql-jdbc.common/handle-additional-options details :separator-style :url))))
(defmethod driver/database-supports? [:clickhouse :uploads] [_driver _feature db]
- (if (:details db)
- (try (cloud? (:details db))
- (catch java.sql.SQLException _e
- false))
- false))
+ (boolean (-> db clickhouse-version/dbms-version :cloud)))
(defmethod driver/can-connect? :clickhouse
[driver details]
@@ -315,10 +287,7 @@
(defmethod driver/database-supports? [:clickhouse :connection-impersonation]
[_driver _feature db]
(if db
- (try (clickhouse-version/is-at-least? 24 4 db)
- (catch Throwable e
- (log/warn e "Error checking connection impersonation")
- false))
+ (clickhouse-version/is-at-least? 24 4 db)
false))
(defmethod driver.sql/set-role-statement :clickhouse
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj
index 17d043efaad..4550a256944 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj
@@ -28,18 +28,29 @@
(sql-jdbc.conn/db->pooled-connection-spec database)
nil
(fn [^java.sql.Connection conn]
- (with-open [stmt (.createStatement conn)
- rset (.executeQuery stmt clickhouse-version-query)]
- (when (.next rset)
- {:version (.getString rset 1)
- :semantic-version {:major (.getInt rset 2)
- :minor (.getInt rset 3)}})))))
+ (with-open [ver-stmt (.createStatement conn)
+ ver-rset (.executeQuery ver-stmt clickhouse-version-query)
+ cloud-stmt (.createStatement conn)
+ cloud-rset (.executeQuery cloud-stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")]
+ (cond-> nil
+ (.next ver-rset)
+ (assoc :version (.getString ver-rset 1)
+ :semantic-version {:major (.getInt ver-rset 2)
+ :minor (.getInt ver-rset 3)})
+
+ (.next cloud-rset)
+ (assoc :cloud (.getBoolean cloud-rset 1)))))))
:ttl/threshold default-cache-ttl))
(defmethod driver/dbms-version :clickhouse
[_driver db]
(get-clickhouse-version db))
+(defn dbms-version
+ "Returns dbms version from a db that may be a snake-hating-map"
+ [db]
+ ((some-fn :dbms-version :dbms_version) db))
+
(defn is-at-least?
"Is ClickHouse version at least `major.minor` (e.g., 24.4)?"
([major minor]
@@ -47,8 +58,7 @@
(is-at-least? major minor (driver-api/database (driver-api/metadata-provider))))
([major minor db]
;; used from the Driver overrides; we have access to the DB object
- (let [version (driver/dbms-version :clickhouse db)
- semantic (:semantic-version version)]
+ (let [semantic (-> db dbms-version :semantic-version)]
(driver.u/semantic-version-gte [(:major semantic) (:minor semantic)] [major minor]))))
(defn with-min
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj
index 8be421365b4..7d386a1f30d 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj
@@ -1,15 +1,14 @@
(ns ^:mb/driver-tests metabase.driver.clickhouse-impersonation-test
"SET ROLE (connection impersonation feature) tests with single node or on-premise cluster setups."
(:require
- [clojure.core.memoize :as memoize]
[clojure.test :refer :all]
[metabase-enterprise.impersonation.util-test :as impersonation.tu]
[metabase.driver :as driver]
- [metabase.driver.clickhouse-version :as clickhouse.version]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.test-util :as driver.tu]
+ [metabase.query-processor :as qp]
[metabase.query-processor.store :as qp.store]
[metabase.sync.core :as sync.core]
[metabase.test :as mt]
@@ -99,7 +98,12 @@
(testing "on-premise cluster"
(testing "should support the impersonation feature"
(t2.with-temp/with-temp
- [:model/Database db {:engine :clickhouse :details {:user "default" :port (mt/db-test-env-var :clickhouse :nginx-port)}}]
+ [:model/Database db {:engine :clickhouse
+ :details {:user "default" :port (mt/db-test-env-var :clickhouse :nginx-port)}
+ :dbms_version {:version "25.1.3.23"
+ :semantic-version {:major 25
+ :minor 1}
+ :cloud false}}]
(is (true? (driver/database-supports? :clickhouse :connection-impersonation db)))))
(let [statements ["CREATE DATABASE IF NOT EXISTS `metabase_test_role_db` ON CLUSTER '{cluster}';"
"CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` ON CLUSTER '{cluster}' (i Int32)
@@ -120,7 +124,13 @@
(testing "older ClickHouse version" ;; 23.3
(testing "should NOT support the impersonation feature"
(t2.with-temp/with-temp
- [:model/Database db {:engine :clickhouse :details {:user "default" :port (mt/db-test-env-var :clickhouse :old-port)}}]
+ [:model/Database db {:engine :clickhouse
+ :details {:user "default" :port (mt/db-test-env-var :clickhouse :old-port)}
+ ;; deliberately an older version here
+ :dbms_version {:version "23.3"
+ :semantic-version {:major 23
+ :minor 3}
+ :cloud false}}]
(is (false? (driver/database-supports? :clickhouse :connection-impersonation db)))))))))
(deftest conn-impersonation-test-clickhouse
@@ -195,25 +205,23 @@
(defmacro ^:private with-ssh-tunnel! [tunnel-details & body]
`(with-ssh-tunnel*! ~tunnel-details (^:once fn* [] ~@body)))
-(deftest impersonation-support-check-over-ssh-tunnel-test
- (testing "driver-supports? :connection-impersonation should work through an SSH tunnel (#62377)"
- ;; Use an in-memory SSH server that forwards to the correct port through localhost.
- ;; Since the real database is still reachable locally, it's possible for the driver to ignore the SSH tunnel and
- ;; still connect to the DB. So this test runs twice, once with all correct and once with an incorrect SSH
- ;; password. The latter check should fail (returning false for `driver-supports?`) if the SSH tunnel is being
- ;; respected.
- (mt/test-driver :clickhouse
- (let [username "username", password "password"]
- (with-open [ssh-server (driver.tu/basic-auth-ssh-server username password)]
- (doseq [[correct-password? ssh-password] [[true password] [false "wrong-password"]]]
- (memoize/memo-clear! @#'clickhouse.version/get-clickhouse-version)
- (let [ssh-port (.getPort ssh-server)]
- (with-ssh-tunnel! {:tunnel-enabled true
- :tunnel-host "localhost"
- :tunnel-auth-option "password"
- :tunnel-port ssh-port
- :tunnel-user username
- :tunnel-pass ssh-password}
- (testing (str "using " ssh-password)
- (is (= correct-password?
- (driver/database-supports? :clickhouse :connection-impersonation (mt/id)))))))))))))
+(deftest clickhouse-ssh-tunnel-test
+ (mt/test-driver :clickhouse
+ (let [username "username"
+ password "password"]
+ (with-open [ssh-server (driver.tu/basic-auth-ssh-server username password)]
+ (let [tunnel-details {:tunnel-enabled true
+ :tunnel-host "localhost"
+ :tunnel-auth-option "password"
+ :tunnel-port (.getPort ssh-server)
+ :tunnel-user username
+ :tunnel-pass password}]
+ (testing "can connect and query through ssh tunnel"
+ (with-ssh-tunnel! tunnel-details
+ (is (= 100
+ (count (mt/rows (qp/process-query (mt/mbql-query venues))))))))
+ (testing "connection fails with wrong ssh credentials"
+ (with-ssh-tunnel! (assoc tunnel-details :tunnel-pass "wrong-password")
+ (is (thrown-with-msg?
+ org.apache.sshd.common.SshException #"No more authentication methods available"
+ (count (mt/rows (qp/process-query (mt/mbql-query venues)))))))))))))
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
index 904cda43911..c35f1a0a276 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
@@ -3,7 +3,6 @@
(:require
[clojure.test :refer :all]
[metabase.driver :as driver]
- [metabase.driver.clickhouse :as clickhouse]
[metabase.driver.clickhouse-qp :as clickhouse-qp]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
@@ -44,84 +43,61 @@
(driver/db-default-timezone :clickhouse spec))))))
(deftest ^:parallel clickhouse-connection-string
- (mt/with-dynamic-fn-redefs [;; This function's implementation requires the connection details to actually connect to the
- ;; database, which is orthogonal to the purpose of this test.
- clickhouse/cloud? (constantly false)]
- (testing "connection with no additional options"
- (is (= ctd/default-connection-params
- (sql-jdbc.conn/connection-details->spec
- :clickhouse
- {}))))
- (testing "custom connection with additional options"
- (is (= (merge
- ctd/default-connection-params
- {:subname "//myclickhouse:9999/foo?sessionTimeout=42"
- :user "bob"
- :password "qaz"
- :ssl true
- :custom_http_params "max_threads=42,allow_experimental_analyzer=0"})
- (sql-jdbc.conn/connection-details->spec
- :clickhouse
- {:host "myclickhouse"
- :port 9999
- :user "bob"
- :password "qaz"
- :dbname "foo"
- :additional-options "sessionTimeout=42"
- :ssl true
- :clickhouse-settings "max_threads=42,allow_experimental_analyzer=0"}))))
- (testing "nil dbname handling"
- (is (= ctd/default-connection-params
- (sql-jdbc.conn/connection-details->spec
- :clickhouse {:dbname nil}))))
- (testing "schema removal"
- (doall
- (for [host ["localhost" "http://localhost" "https://localhost"]]
- (testing (str "for host " host)
- (is (= ctd/default-connection-params
- (sql-jdbc.conn/connection-details->spec
- :clickhouse {:host host}))))))
- (doall
- (for [host ["myhost" "http://myhost" "https://myhost"]]
- (testing (str "for host " host)
- (is (= (merge ctd/default-connection-params
- {:subname "//myhost:8123/default"})
- (sql-jdbc.conn/connection-details->spec
- :clickhouse {:host host}))))))
- (doall
- (for [host ["sub.example.com" "http://sub.example.com" "https://sub.example.com"]]
- (testing (str "for host " host " with some additional params")
- (is (= (merge ctd/default-connection-params
- {:subname "//sub.example.com:8443/mydb" :ssl true})
- (sql-jdbc.conn/connection-details->spec
- :clickhouse {:host host :dbname "mydb" :port 8443 :ssl true})))))))))
+ (testing "connection with no additional options"
+ (is (= ctd/default-connection-params
+ (sql-jdbc.conn/connection-details->spec
+ :clickhouse
+ {}))))
+ (testing "custom connection with additional options"
+ (is (= (merge
+ ctd/default-connection-params
+ {:subname "//myclickhouse:9999/foo?sessionTimeout=42"
+ :user "bob"
+ :password "qaz"
+ :ssl true
+ :custom_http_params "max_threads=42,allow_experimental_analyzer=0"})
+ (sql-jdbc.conn/connection-details->spec
+ :clickhouse
+ {:host "myclickhouse"
+ :port 9999
+ :user "bob"
+ :password "qaz"
+ :dbname "foo"
+ :additional-options "sessionTimeout=42"
+ :ssl true
+ :clickhouse-settings "max_threads=42,allow_experimental_analyzer=0"}))))
+ (testing "nil dbname handling"
+ (is (= ctd/default-connection-params
+ (sql-jdbc.conn/connection-details->spec
+ :clickhouse {:dbname nil}))))
+ (testing "schema removal"
+ (doall
+ (for [host ["localhost" "http://localhost" "https://localhost"]]
+ (testing (str "for host " host)
+ (is (= ctd/default-connection-params
+ (sql-jdbc.conn/connection-details->spec
+ :clickhouse {:host host}))))))
+ (doall
+ (for [host ["myhost" "http://myhost" "https://myhost"]]
+ (testing (str "for host " host)
+ (is (= (merge ctd/default-connection-params
+ {:subname "//myhost:8123/default"})
+ (sql-jdbc.conn/connection-details->spec
+ :clickhouse {:host host}))))))
+ (doall
+ (for [host ["sub.example.com" "http://sub.example.com" "https://sub.example.com"]]
+ (testing (str "for host " host " with some additional params")
+ (is (= (merge ctd/default-connection-params
+ {:subname "//sub.example.com:8443/mydb" :ssl true})
+ (sql-jdbc.conn/connection-details->spec
+ :clickhouse {:host host :dbname "mydb" :port 8443 :ssl true}))))))))
(deftest ^:parallel clickhouse-connection-string-select-sequential-consistency
- (mt/with-dynamic-fn-redefs [;; This function's implementation requires the connection details to actually
- ;; connect to the database, which is orthogonal to the purpose of this test.
- clickhouse/cloud? (constantly true)]
- (testing "connection with no additional options"
- (is (= (assoc ctd/default-connection-params :select_sequential_consistency true)
- (sql-jdbc.conn/connection-details->spec
- :clickhouse
- {}))))))
-
-(deftest clickhouse-connection-fails-test
- (mt/test-driver :clickhouse
- (mt/with-temp [:model/Database db {:details (assoc (:details (mt/db)) :password "wrongpassword") :engine :clickhouse}]
- (testing "sense check that checking the cloud mode fails with a SQLException."
- ;; nil arg isn't tested here, as it will pick up the defaults, which is the same as the Docker instance credentials.
- (is (thrown? java.sql.SQLException (#'clickhouse/cloud? (:details db)))))
- (testing "`driver/database-supports? :uploads` does not throw even if the connection fails."
- (is (false? (driver/database-supports? :clickhouse :uploads db)))
- (is (false? (driver/database-supports? :clickhouse :uploads nil))))
- (testing "`driver/database-supports? :connection-impersonation` does not throw even if the connection fails."
- (is (false? (driver/database-supports? :clickhouse :connection-impersonation db)))
- (is (false? (driver/database-supports? :clickhouse :connection-impersonation nil))))
- (testing (str "`sql-jdbc.conn/connection-details->spec` does not throw even if the connection fails, "
- "and doesn't include the `select_sequential_consistency` parameter.")
- (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse db))))
- (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse nil))))))))
+ (testing "connection with no additional options"
+ (is (= (assoc ctd/default-connection-params :select_sequential_consistency true)
+ (sql-jdbc.conn/connection-details->spec
+ :clickhouse
+ {})))))
(deftest ^:parallel clickhouse-tls
(mt/test-driver :clickhouse
@@ -358,6 +334,12 @@
"Failed to get server info"
"Code: 516. DB::Exception: asdf: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) (version 25.7.4.11 (official build))"]))))
+(deftest ^:parallel uploads-supported-test
+ (mt/test-driver :clickhouse
+ (is (false? (driver/database-supports? driver/*driver* :uploads (mt/db))))
+ (is (true? (driver/database-supports? driver/*driver* :uploads (assoc-in (mt/db) [:dbms-version :cloud] true))))
+ (is (true? (driver/database-supports? driver/*driver* :uploads (assoc-in (mt/db) [:dbms_version :cloud] true))))))
+
(deftest ^:parallel type->database-type-test
(testing "type->database-type multimethod returns correct ClickHouse types"
(are [base-type expected] (= expected (driver/type->database-type :clickhouse base-type))
diff --git a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
index 9e3b3a8caea..594717a07a7 100644
--- a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
+++ b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
@@ -56,7 +56,8 @@
:max_open_connections 100
:remember_last_set_roles true
:http_connection_provider "HTTP_URL_CONNECTION"
- :custom_http_params ""})
+ :custom_http_params ""
+ :select_sequential_consistency true})
(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Boolean] [_ _] "Boolean")
(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/BigInteger] [_ _] "Int64")