The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix ssh tunneling for clickhouse (#63464)
Eric Normand6afcf2aed582
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.cljindex dff992030f4..f82df5d5ab4 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj@@ -1,7 +1,6 @@(ns metabase.driver.clickhouse"Driver for ClickHouse databases"(:require- [clojure.core.memoize :as memoize][clojure.java.jdbc :as jdbc][clojure.string :as str][metabase.driver :as driver]@@ -68,33 +67,6 @@(def ^:private default-connection-details{:user "default" :password "" :dbname "default" :host "localhost" :port 8123})-(defn- connection-details->spec* [details]- (let [;; ensure defaults merge on top of nils- details (reduce-kv (fn [m k v] (assoc m k (or v (k default-connection-details))))- default-connection-details- details)- {:keys [user password dbname host port ssl clickhouse-settings max-open-connections]} details- host (cond ; JDBCv1 used to accept schema in the `host` configuration option- (str/starts-with? host "http://") (subs host 7)- (str/starts-with? host "https://") (subs host 8)- :else host)]- (-> {:classname "com.clickhouse.jdbc.ClickHouseDriver"- :subprotocol "clickhouse"- :subname (str "//" host ":" port "/" dbname)- :password (or password "")- :user user- :ssl (boolean ssl)- :use_server_time_zone_for_dates true- :product_name (format "metabase/%s" (:tag driver-api/mb-version-info))- :remember_last_set_roles true- :http_connection_provider "HTTP_URL_CONNECTION"- :jdbc_ignore_unsupported_values "true"- :jdbc_schema_term "schema"- :max_open_connections (or max-open-connections 100)- ;; see also: https://clickhouse.com/docs/en/integrations/java#configuration- :custom_http_params (or clickhouse-settings "")}- (sql-jdbc.common/handle-additional-options details :separator-style :url))))-(defmethod sql-jdbc.execute/do-with-connection-with-options :clickhouse[driver db-or-id-or-spec {:keys [^String session-timezone _write?] :as options} f](sql-jdbc.execute/do-with-resolved-connection@@ -122,37 +94,37 @@(sql-jdbc.execute/set-time-zone-if-supported! driver conn session-timezone))(f conn))))-(def ^:private ^{:arglists '([db-details])} cloud?- "Returns true if the `db-details` are for a ClickHouse Cloud instance, and false otherwise. If it fails to connect- to the database, it throws a java.sql.SQLException."- (memoize/ttl- (fn [db-details]- (let [spec (connection-details->spec* db-details)]- (sql-jdbc.execute/do-with-connection-with-options- :clickhouse spec nil- (fn [^java.sql.Connection conn]- (with-open [stmt (.createStatement conn)- rset (.executeQuery stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")]- (if (.next rset) (.getBoolean rset 1) false))))))- ;; cache the results for 48 hours; TTL is here only to eventually clear out old entries- :ttl/threshold (* 48 60 60 1000)))-(defmethod sql-jdbc.conn/connection-details->spec :clickhouse[_ details]- (cond-> (connection-details->spec* details)- (try (cloud? details)- (catch java.sql.SQLException _e- false))- ;; select_sequential_consistency guarantees that we can query data from any replica in CH Cloud- ;; immediately after it is written- (assoc :select_sequential_consistency true)))+ (let [;; ensure defaults merge on top of nils+ details (reduce-kv (fn [m k v] (assoc m k (or v (k default-connection-details))))+ default-connection-details+ details)+ {:keys [user password dbname host port ssl clickhouse-settings max-open-connections]} details+ host (cond ; JDBCv1 used to accept schema in the `host` configuration option+ (str/starts-with? host "http://") (subs host 7)+ (str/starts-with? host "https://") (subs host 8)+ :else host)]+ (-> {:classname "com.clickhouse.jdbc.ClickHouseDriver"+ :subprotocol "clickhouse"+ :subname (str "//" host ":" port "/" dbname)+ :password (or password "")+ :user user+ :ssl (boolean ssl)+ :use_server_time_zone_for_dates true+ :product_name (format "metabase/%s" (:tag driver-api/mb-version-info))+ :remember_last_set_roles true+ :http_connection_provider "HTTP_URL_CONNECTION"+ :jdbc_ignore_unsupported_values "true"+ :jdbc_schema_term "schema"+ :select_sequential_consistency true+ :max_open_connections (or max-open-connections 100)+ ;; see also: https://clickhouse.com/docs/en/integrations/java#configuration+ :custom_http_params (or clickhouse-settings "")}+ (sql-jdbc.common/handle-additional-options details :separator-style :url))))(defmethod driver/database-supports? [:clickhouse :uploads] [_driver _feature db]- (if (:details db)- (try (cloud? (:details db))- (catch java.sql.SQLException _e- false))- false))+ (boolean (-> db clickhouse-version/dbms-version :cloud)))(defmethod driver/can-connect? :clickhouse[driver details]@@ -315,10 +287,7 @@(defmethod driver/database-supports? [:clickhouse :connection-impersonation][_driver _feature db](if db- (try (clickhouse-version/is-at-least? 24 4 db)- (catch Throwable e- (log/warn e "Error checking connection impersonation")- false))+ (clickhouse-version/is-at-least? 24 4 db)false))(defmethod driver.sql/set-role-statement :clickhousediff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.cljindex 17d043efaad..4550a256944 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj@@ -28,18 +28,29 @@(sql-jdbc.conn/db->pooled-connection-spec database)nil(fn [^java.sql.Connection conn]- (with-open [stmt (.createStatement conn)- rset (.executeQuery stmt clickhouse-version-query)]- (when (.next rset)- {:version (.getString rset 1)- :semantic-version {:major (.getInt rset 2)- :minor (.getInt rset 3)}})))))+ (with-open [ver-stmt (.createStatement conn)+ ver-rset (.executeQuery ver-stmt clickhouse-version-query)+ cloud-stmt (.createStatement conn)+ cloud-rset (.executeQuery cloud-stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")]+ (cond-> nil+ (.next ver-rset)+ (assoc :version (.getString ver-rset 1)+ :semantic-version {:major (.getInt ver-rset 2)+ :minor (.getInt ver-rset 3)})++ (.next cloud-rset)+ (assoc :cloud (.getBoolean cloud-rset 1))))))):ttl/threshold default-cache-ttl))(defmethod driver/dbms-version :clickhouse[_driver db](get-clickhouse-version db))+(defn dbms-version+ "Returns dbms version from a db that may be a snake-hating-map"+ [db]+ ((some-fn :dbms-version :dbms_version) db))+(defn is-at-least?"Is ClickHouse version at least `major.minor` (e.g., 24.4)?"([major minor]@@ -47,8 +58,7 @@(is-at-least? major minor (driver-api/database (driver-api/metadata-provider))))([major minor db];; used from the Driver overrides; we have access to the DB object- (let [version (driver/dbms-version :clickhouse db)- semantic (:semantic-version version)]+ (let [semantic (-> db dbms-version :semantic-version)](driver.u/semantic-version-gte [(:major semantic) (:minor semantic)] [major minor]))))(defn with-mindiff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.cljindex 8be421365b4..7d386a1f30d 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj@@ -1,15 +1,14 @@(ns ^:mb/driver-tests metabase.driver.clickhouse-impersonation-test"SET ROLE (connection impersonation feature) tests with single node or on-premise cluster setups."(:require- [clojure.core.memoize :as memoize][clojure.test :refer :all][metabase-enterprise.impersonation.util-test :as impersonation.tu][metabase.driver :as driver]- [metabase.driver.clickhouse-version :as clickhouse.version][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.test-util :as driver.tu]+ [metabase.query-processor :as qp][metabase.query-processor.store :as qp.store][metabase.sync.core :as sync.core][metabase.test :as mt]@@ -99,7 +98,12 @@(testing "on-premise cluster"(testing "should support the impersonation feature"(t2.with-temp/with-temp- [:model/Database db {:engine :clickhouse :details {:user "default" :port (mt/db-test-env-var :clickhouse :nginx-port)}}]+ [:model/Database db {:engine :clickhouse+ :details {:user "default" :port (mt/db-test-env-var :clickhouse :nginx-port)}+ :dbms_version {:version "25.1.3.23"+ :semantic-version {:major 25+ :minor 1}+ :cloud false}}](is (true? (driver/database-supports? :clickhouse :connection-impersonation db)))))(let [statements ["CREATE DATABASE IF NOT EXISTS `metabase_test_role_db` ON CLUSTER '{cluster}';""CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` ON CLUSTER '{cluster}' (i Int32)@@ -120,7 +124,13 @@(testing "older ClickHouse version" ;; 23.3(testing "should NOT support the impersonation feature"(t2.with-temp/with-temp- [:model/Database db {:engine :clickhouse :details {:user "default" :port (mt/db-test-env-var :clickhouse :old-port)}}]+ [:model/Database db {:engine :clickhouse+ :details {:user "default" :port (mt/db-test-env-var :clickhouse :old-port)}+ ;; deliberately an older version here+ :dbms_version {:version "23.3"+ :semantic-version {:major 23+ :minor 3}+ :cloud false}}](is (false? (driver/database-supports? :clickhouse :connection-impersonation db)))))))))(deftest conn-impersonation-test-clickhouse@@ -195,25 +205,23 @@(defmacro ^:private with-ssh-tunnel! [tunnel-details & body]`(with-ssh-tunnel*! ~tunnel-details (^:once fn* [] ~@body)))-(deftest impersonation-support-check-over-ssh-tunnel-test- (testing "driver-supports? :connection-impersonation should work through an SSH tunnel (#62377)"- ;; Use an in-memory SSH server that forwards to the correct port through localhost.- ;; Since the real database is still reachable locally, it's possible for the driver to ignore the SSH tunnel and- ;; still connect to the DB. So this test runs twice, once with all correct and once with an incorrect SSH- ;; password. The latter check should fail (returning false for `driver-supports?`) if the SSH tunnel is being- ;; respected.- (mt/test-driver :clickhouse- (let [username "username", password "password"]- (with-open [ssh-server (driver.tu/basic-auth-ssh-server username password)]- (doseq [[correct-password? ssh-password] [[true password] [false "wrong-password"]]]- (memoize/memo-clear! @#'clickhouse.version/get-clickhouse-version)- (let [ssh-port (.getPort ssh-server)]- (with-ssh-tunnel! {:tunnel-enabled true- :tunnel-host "localhost"- :tunnel-auth-option "password"- :tunnel-port ssh-port- :tunnel-user username- :tunnel-pass ssh-password}- (testing (str "using " ssh-password)- (is (= correct-password?- (driver/database-supports? :clickhouse :connection-impersonation (mt/id)))))))))))))+(deftest clickhouse-ssh-tunnel-test+ (mt/test-driver :clickhouse+ (let [username "username"+ password "password"]+ (with-open [ssh-server (driver.tu/basic-auth-ssh-server username password)]+ (let [tunnel-details {:tunnel-enabled true+ :tunnel-host "localhost"+ :tunnel-auth-option "password"+ :tunnel-port (.getPort ssh-server)+ :tunnel-user username+ :tunnel-pass password}]+ (testing "can connect and query through ssh tunnel"+ (with-ssh-tunnel! tunnel-details+ (is (= 100+ (count (mt/rows (qp/process-query (mt/mbql-query venues))))))))+ (testing "connection fails with wrong ssh credentials"+ (with-ssh-tunnel! (assoc tunnel-details :tunnel-pass "wrong-password")+ (is (thrown-with-msg?+ org.apache.sshd.common.SshException #"No more authentication methods available"+ (count (mt/rows (qp/process-query (mt/mbql-query venues)))))))))))))diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.cljindex 904cda43911..c35f1a0a276 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj@@ -3,7 +3,6 @@(:require[clojure.test :refer :all][metabase.driver :as driver]- [metabase.driver.clickhouse :as clickhouse][metabase.driver.clickhouse-qp :as clickhouse-qp][metabase.driver.sql-jdbc :as sql-jdbc][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]@@ -44,84 +43,61 @@(driver/db-default-timezone :clickhouse spec))))))(deftest ^:parallel clickhouse-connection-string- (mt/with-dynamic-fn-redefs [;; This function's implementation requires the connection details to actually connect to the- ;; database, which is orthogonal to the purpose of this test.- clickhouse/cloud? (constantly false)]- (testing "connection with no additional options"- (is (= ctd/default-connection-params- (sql-jdbc.conn/connection-details->spec- :clickhouse- {}))))- (testing "custom connection with additional options"- (is (= (merge- ctd/default-connection-params- {:subname "//myclickhouse:9999/foo?sessionTimeout=42"- :user "bob"- :password "qaz"- :ssl true- :custom_http_params "max_threads=42,allow_experimental_analyzer=0"})- (sql-jdbc.conn/connection-details->spec- :clickhouse- {:host "myclickhouse"- :port 9999- :user "bob"- :password "qaz"- :dbname "foo"- :additional-options "sessionTimeout=42"- :ssl true- :clickhouse-settings "max_threads=42,allow_experimental_analyzer=0"}))))- (testing "nil dbname handling"- (is (= ctd/default-connection-params- (sql-jdbc.conn/connection-details->spec- :clickhouse {:dbname nil}))))- (testing "schema removal"- (doall- (for [host ["localhost" "http://localhost" "https://localhost"]]- (testing (str "for host " host)- (is (= ctd/default-connection-params- (sql-jdbc.conn/connection-details->spec- :clickhouse {:host host}))))))- (doall- (for [host ["myhost" "http://myhost" "https://myhost"]]- (testing (str "for host " host)- (is (= (merge ctd/default-connection-params- {:subname "//myhost:8123/default"})- (sql-jdbc.conn/connection-details->spec- :clickhouse {:host host}))))))- (doall- (for [host ["sub.example.com" "http://sub.example.com" "https://sub.example.com"]]- (testing (str "for host " host " with some additional params")- (is (= (merge ctd/default-connection-params- {:subname "//sub.example.com:8443/mydb" :ssl true})- (sql-jdbc.conn/connection-details->spec- :clickhouse {:host host :dbname "mydb" :port 8443 :ssl true})))))))))+ (testing "connection with no additional options"+ (is (= ctd/default-connection-params+ (sql-jdbc.conn/connection-details->spec+ :clickhouse+ {}))))+ (testing "custom connection with additional options"+ (is (= (merge+ ctd/default-connection-params+ {:subname "//myclickhouse:9999/foo?sessionTimeout=42"+ :user "bob"+ :password "qaz"+ :ssl true+ :custom_http_params "max_threads=42,allow_experimental_analyzer=0"})+ (sql-jdbc.conn/connection-details->spec+ :clickhouse+ {:host "myclickhouse"+ :port 9999+ :user "bob"+ :password "qaz"+ :dbname "foo"+ :additional-options "sessionTimeout=42"+ :ssl true+ :clickhouse-settings "max_threads=42,allow_experimental_analyzer=0"}))))+ (testing "nil dbname handling"+ (is (= ctd/default-connection-params+ (sql-jdbc.conn/connection-details->spec+ :clickhouse {:dbname nil}))))+ (testing "schema removal"+ (doall+ (for [host ["localhost" "http://localhost" "https://localhost"]]+ (testing (str "for host " host)+ (is (= ctd/default-connection-params+ (sql-jdbc.conn/connection-details->spec+ :clickhouse {:host host}))))))+ (doall+ (for [host ["myhost" "http://myhost" "https://myhost"]]+ (testing (str "for host " host)+ (is (= (merge ctd/default-connection-params+ {:subname "//myhost:8123/default"})+ (sql-jdbc.conn/connection-details->spec+ :clickhouse {:host host}))))))+ (doall+ (for [host ["sub.example.com" "http://sub.example.com" "https://sub.example.com"]]+ (testing (str "for host " host " with some additional params")+ (is (= (merge ctd/default-connection-params+ {:subname "//sub.example.com:8443/mydb" :ssl true})+ (sql-jdbc.conn/connection-details->spec+ :clickhouse {:host host :dbname "mydb" :port 8443 :ssl true}))))))))(deftest ^:parallel clickhouse-connection-string-select-sequential-consistency- (mt/with-dynamic-fn-redefs [;; This function's implementation requires the connection details to actually- ;; connect to the database, which is orthogonal to the purpose of this test.- clickhouse/cloud? (constantly true)]- (testing "connection with no additional options"- (is (= (assoc ctd/default-connection-params :select_sequential_consistency true)- (sql-jdbc.conn/connection-details->spec- :clickhouse- {}))))))--(deftest clickhouse-connection-fails-test- (mt/test-driver :clickhouse- (mt/with-temp [:model/Database db {:details (assoc (:details (mt/db)) :password "wrongpassword") :engine :clickhouse}]- (testing "sense check that checking the cloud mode fails with a SQLException."- ;; nil arg isn't tested here, as it will pick up the defaults, which is the same as the Docker instance credentials.- (is (thrown? java.sql.SQLException (#'clickhouse/cloud? (:details db)))))- (testing "`driver/database-supports? :uploads` does not throw even if the connection fails."- (is (false? (driver/database-supports? :clickhouse :uploads db)))- (is (false? (driver/database-supports? :clickhouse :uploads nil))))- (testing "`driver/database-supports? :connection-impersonation` does not throw even if the connection fails."- (is (false? (driver/database-supports? :clickhouse :connection-impersonation db)))- (is (false? (driver/database-supports? :clickhouse :connection-impersonation nil))))- (testing (str "`sql-jdbc.conn/connection-details->spec` does not throw even if the connection fails, "- "and doesn't include the `select_sequential_consistency` parameter.")- (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse db))))- (is (nil? (:select_sequential_consistency (sql-jdbc.conn/connection-details->spec :clickhouse nil))))))))+ (testing "connection with no additional options"+ (is (= (assoc ctd/default-connection-params :select_sequential_consistency true)+ (sql-jdbc.conn/connection-details->spec+ :clickhouse+ {})))))(deftest ^:parallel clickhouse-tls(mt/test-driver :clickhouse@@ -358,6 +334,12 @@"Failed to get server info""Code: 516. DB::Exception: asdf: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) (version 25.7.4.11 (official build))"]))))+(deftest ^:parallel uploads-supported-test+ (mt/test-driver :clickhouse+ (is (false? (driver/database-supports? driver/*driver* :uploads (mt/db))))+ (is (true? (driver/database-supports? driver/*driver* :uploads (assoc-in (mt/db) [:dbms-version :cloud] true))))+ (is (true? (driver/database-supports? driver/*driver* :uploads (assoc-in (mt/db) [:dbms_version :cloud] true))))))+(deftest ^:parallel type->database-type-test(testing "type->database-type multimethod returns correct ClickHouse types"(are [base-type expected] (= expected (driver/type->database-type :clickhouse base-type))diff --git a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.cljindex 9e3b3a8caea..594717a07a7 100644--- a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj+++ b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj@@ -56,7 +56,8 @@:max_open_connections 100:remember_last_set_roles true:http_connection_provider "HTTP_URL_CONNECTION"- :custom_http_params ""})+ :custom_http_params ""+ :select_sequential_consistency true})(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Boolean] [_ _] "Boolean")(defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/BigInteger] [_ _] "Int64")