Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

CSV uploads for non-cloud clickhouse (#83878)

Riley Thompson02d40a0aabea
diff --git a/docs/databases/connections/clickhouse.md b/docs/databases/connections/clickhouse.md
index 1127a0222c2..20310ede5a3 100644
--- a/docs/databases/connections/clickhouse.md
+++ b/docs/databases/connections/clickhouse.md
@@ -105,6 +105,12 @@ A fingerprinting query examines the first 10,000 rows from each column and uses
Set up an additional connection used for write operations. See [Writable connections](../writable-connection.md).
+## Uploads
+
+Metabase supports [CSV uploads](../uploads.md) to ClickHouse Cloud and self-hosted single-node ClickHouse deployments. Self-hosted clusters aren't supported.
+
+For self-hosted deployments, the database user must be able to read the `system.clusters` table, as well as create tables and insert data.
+
## Model features
There aren't (yet) any model features for ClickHouse.
diff --git a/docs/databases/uploads.md b/docs/databases/uploads.md
index 3c08620fa08..cb0ecd15a2f 100644
--- a/docs/databases/uploads.md
+++ b/docs/databases/uploads.md
@@ -21,7 +21,7 @@ To manage upload settings, admins can hit cmd/ctrl + K and search for "Settings
- [MySQL](../databases/connections/mysql.md)
- [Snowflake](../databases/connections/snowflake.md)
- [Redshift](../databases/connections/redshift.md)
-- [ClickHouse](../databases/connections/clickhouse.md) (only supported on ClickHouse Cloud)
+- [ClickHouse](../databases/connections/clickhouse.md#uploads) (ClickHouse Cloud and self-hosted single-node deployments)
## Setting up uploads
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/upload_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/upload_test.clj
index 2baf6262ca0..aeee258c378 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/upload_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/upload_test.clj
@@ -43,7 +43,7 @@
(upload-test/with-uploads-enabled!
(mt/dataset dataset
(mt/with-temp [:model/Collection collection {}
- :model/Database {db-id :id} {:engine driver/*driver* :details (:details (mt/db))}
+ :model/Database {db-id :id} {:engine driver/*driver* :details (:details (mt/db)) :dbms_version (:dbms_version (mt/db))}
:model/Table {table-id :id} {:db_id db-id
:is_upload true}
:model/Card {card-id :id
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
index 15315877cd1..1da981ecd48 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
@@ -3,6 +3,7 @@
(:require
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
+ [honey.sql :as sql]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
[metabase.driver.clickhouse-introspection]
@@ -16,6 +17,8 @@
[metabase.driver.sql-jdbc.common :as sql-jdbc.common]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
+ [metabase.driver.sql-jdbc.quoting :refer [dot-qualified quote-identifier with-quoting]]
+ [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.util :as sql.u]
[metabase.util :as u]
@@ -186,7 +189,8 @@
(sql-jdbc.common/handle-additional-options details :separator-style :url))))
(defmethod driver/database-supports? [:clickhouse :uploads] [_driver _feature db]
- (boolean (-> db clickhouse-version/dbms-version :cloud)))
+ (let [{:keys [cloud single-node]} (clickhouse-version/dbms-version db)]
+ (boolean (or cloud single-node))))
(defmethod driver/can-connect? :clickhouse
[driver details]
@@ -254,6 +258,20 @@
:metabase.upload/datetime "Nullable(DateTime64(3))"
:metabase.upload/offset-datetime nil))
+;; ClickHouse changes a column's type with MODIFY COLUMN; the default ALTER COLUMN is a syntax error.
+(defmethod sql-jdbc.sync/alter-table-columns-sql :clickhouse
+ [driver table-name column-definitions & _opts]
+ (with-quoting driver
+ (first (sql/format {:alter-table (dot-qualified table-name)
+ :modify-column (map (fn [[column-name type-and-constraints]]
+ (into [(quote-identifier column-name)]
+ (if (string? type-and-constraints)
+ [[:raw type-and-constraints]]
+ type-and-constraints)))
+ column-definitions)}
+ :quoted true
+ :dialect (sql.qp/quote-style driver)))))
+
(defmulti ^:private type->database-type
"Internal type->database-type multimethod for ClickHouse that dispatches on type."
{:arglists '([type])}
@@ -278,9 +296,10 @@
(defmethod driver/table-name-length-limit :clickhouse
[_driver]
- ;; FIXME: This is a lie because you're really limited by a filesystems' limits, because Clickhouse uses
- ;; filenames as table/column names. But its an approximation
- 206)
+ ;; ClickHouse stores table metadata as files, so the real limit is on the combined length of the database and table
+ ;; names: on recent versions `max table name length = 213 - (length of the database name)`, so the old value of 206
+ ;; only held for a 7-character database such as `default`. 128 leaves room for database names up to 85 characters.
+ 128)
(defn- escape-ident
;; Backslash-escape rather than double the backtick: ClickHouse identifiers follow string-literal escaping, where
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
index 20bd4b86111..9513ce58bb3 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj
@@ -584,7 +584,7 @@
(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/BIGINT]
[_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i]
(fn []
- (with-null-check rs (.getBigDecimal rs i))))
+ (with-null-check rs (.getLong rs i))))
(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/INTEGER]
[_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i]
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj
index 1677b19ceaa..b2ed976c1f1 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj
@@ -8,7 +8,10 @@
[metabase.driver.connection :as driver.conn]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
- [metabase.driver.util :as driver.u]))
+ [metabase.driver.util :as driver.u]
+ [metabase.util.log :as log])
+ (:import
+ (java.sql Connection SQLException)))
(set! *warn-on-reflection* true)
@@ -20,6 +23,18 @@
(str "WITH s AS (SELECT version() AS ver, splitByChar('.', ver) AS verSplit) "
"SELECT s.ver, toInt32(verSplit[1]), toInt32(verSplit[2]) FROM s"))
+(defn- single-node?
+ "Return true if system.clusters has no entries marked as non-local.
+ This check uses configuration data and does not verify the number of active nodes."
+ [^Connection conn]
+ (try
+ (with-open [stmt (.createStatement conn)
+ rset (.executeQuery stmt "SELECT count() = 0 FROM system.clusters WHERE is_local = 0")]
+ (and (.next rset) (.getBoolean rset 1)))
+ (catch SQLException e
+ (log/warn e "Could not determine whether ClickHouse is a single-node deployment.")
+ false)))
+
(def ^:private ^{:arglists '([database])} get-clickhouse-version
(memoize/ttl
^{:clojure.core.memoize/args-fn (fn [[database]] (driver.conn/effective-details database))}
@@ -33,14 +48,15 @@
ver-rset (.executeQuery ver-stmt clickhouse-version-query)
cloud-stmt (.createStatement conn)
cloud-rset (.executeQuery cloud-stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")]
- (cond-> nil
- (.next ver-rset)
- (assoc :version (.getString ver-rset 1)
- :semantic-version {:major (.getInt ver-rset 2)
- :minor (.getInt ver-rset 3)})
+ (let [cloud? (and (.next cloud-rset) (.getBoolean cloud-rset 1))]
+ (cond-> {:cloud cloud?}
+ (.next ver-rset)
+ (assoc :version (.getString ver-rset 1)
+ :semantic-version {:major (.getInt ver-rset 2)
+ :minor (.getInt ver-rset 3)})
- (.next cloud-rset)
- (assoc :cloud (.getBoolean cloud-rset 1)))))))
+ (not cloud?)
+ (assoc :single-node (single-node? conn))))))))
:ttl/threshold default-cache-ttl))
(defmethod driver/dbms-version :clickhouse
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
index 16b3a2c6c7a..28e53e28c85 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
@@ -8,7 +8,6 @@
[metabase.driver :as driver]
[metabase.driver.clickhouse :as clickhouse]
[metabase.driver.clickhouse-qp :as clickhouse-qp]
- [metabase.driver.clickhouse-version :as clickhouse-version]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
@@ -728,54 +727,53 @@
(deftest ^:parallel uploads-supported-test
(mt/test-driver :clickhouse
- (is (false? (driver/database-supports? driver/*driver* :uploads (mt/db))))
+ (is (true? (driver/database-supports? driver/*driver* :uploads (mt/db))))
(is (true? (driver/database-supports? driver/*driver* :uploads (assoc-in (mt/db) [:dbms-version :cloud] true))))
(is (true? (driver/database-supports? driver/*driver* :uploads (assoc-in (mt/db) [:dbms_version :cloud] true))))))
(deftest ^:synchronized csv-upload-and-sync-test
- (testing "ClickHouse CSV uploads work correctly when cloud mode is enabled"
+ (testing "ClickHouse CSV uploads work correctly"
(mt/test-driver :clickhouse
- (mt/with-dynamic-fn-redefs [clickhouse-version/dbms-version (constantly {:cloud true
- :version "24.8.1"
- :semantic-version {:major 24 :minor 8}})]
- (let [details (-> (mt/dbdef->connection-details :clickhouse :db {:database-name "uploads_schema"})
- (assoc :enable-multiple-db false))
- conn-spec (sql-jdbc.conn/connection-details->spec :clickhouse details)]
- (driver/create-schema-if-needed! :clickhouse conn-spec "uploads_schema")
- (try
- (mt/with-temp [:model/Database db {:engine :clickhouse
- :details details}]
- (is (true? (driver/database-supports? :clickhouse :uploads db)))
- (testing "an upload schema is required"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"A schema has not been set."
- (upload-test/do-with-uploaded-example-csv!
- {:db-id (:id db)
- :auxiliary-sync-steps :synchronous
- :schema-name ""}
- identity))))
- (testing "upload models work after sync"
- (upload-test/do-with-uploaded-example-csv!
- {:db-id (:id db)
- :auxiliary-sync-steps :synchronous
- :schema-name "uploads_schema"}
- (fn [model]
- (let [query-model (fn []
- (let [mp (lib-be/application-database-metadata-provider (:id db))
- card (lib.metadata/card mp (:id model))]
- (->> (lib/query mp card)
- (qp/process-query)
- (mt/formatted-rows [int str]))))]
- (is (= [[1 " Luke Skywalker"]
- [2 " Darth Vader"]]
- (query-model)))
- (sync/sync-database! db {:scan :schema})
- (is (= [[1 " Luke Skywalker"]
- [2 " Darth Vader"]]
- (query-model))))))))
- (finally
- (jdbc/execute! conn-spec ["DROP DATABASE IF EXISTS `uploads_schema`"]))))))))
+ (let [details (-> (mt/dbdef->connection-details :clickhouse :db {:database-name "uploads_schema"})
+ (assoc :enable-multiple-db false))
+ dbms-version (driver/dbms-version :clickhouse (mt/db))
+ conn-spec (sql-jdbc.conn/connection-details->spec :clickhouse details)]
+ (driver/create-schema-if-needed! :clickhouse conn-spec "uploads_schema")
+ (try
+ (mt/with-temp [:model/Database db {:engine :clickhouse
+ :details details
+ :dbms_version dbms-version}]
+ (is (true? (driver/database-supports? :clickhouse :uploads db)))
+ (testing "an upload schema is required"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"A schema has not been set."
+ (upload-test/do-with-uploaded-example-csv!
+ {:db-id (:id db)
+ :auxiliary-sync-steps :synchronous
+ :schema-name ""}
+ identity))))
+ (testing "upload models work after sync"
+ (upload-test/do-with-uploaded-example-csv!
+ {:db-id (:id db)
+ :auxiliary-sync-steps :synchronous
+ :schema-name "uploads_schema"}
+ (fn [model]
+ (let [query-model (fn []
+ (let [mp (lib-be/application-database-metadata-provider (:id db))
+ card (lib.metadata/card mp (:id model))]
+ (->> (lib/query mp card)
+ (qp/process-query)
+ (mt/formatted-rows [int str]))))]
+ (is (= [[1 " Luke Skywalker"]
+ [2 " Darth Vader"]]
+ (query-model)))
+ (sync/sync-database! db {:scan :schema})
+ (is (= [[1 " Luke Skywalker"]
+ [2 " Darth Vader"]]
+ (query-model))))))))
+ (finally
+ (jdbc/execute! conn-spec ["DROP DATABASE IF EXISTS `uploads_schema`"])))))))
(deftest ^:parallel type->database-type-test
(testing "type->database-type multimethod returns correct ClickHouse types"
diff --git a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
index df6097df561..e28bb96cb09 100644
--- a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
+++ b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj
@@ -191,7 +191,7 @@
(defmethod sql.tx/add-fk-sql :clickhouse [& _] nil)
-(defmethod sql.tx/session-schema :clickhouse [_] "default")
+(defmethod sql.tx/session-schema :clickhouse [_] (-> (mt/db) :details :db))
(defmethod sql.tx/generated-column-sql :clickhouse [_ expr]
(format "ALIAS (%s)" expr))
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index 88daf475d98..0b17dc7bbd9 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -1095,7 +1095,8 @@
[:semantic-version {:optional true} [:maybe [:or
[:sequential :int]
[:map {:closed true} [:major :int] [:minor :int]]]]]
- [:cloud {:optional true} [:maybe :boolean]]]]]
+ [:cloud {:optional true} [:maybe :boolean]]
+ [:single-node {:optional true} [:maybe :boolean]]]]]
[:details {:optional true} ::lib.schema.common/database-details]
[:engine {:optional true} [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]]
[:features {:optional true} [:set [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]]]
diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.clj
index 05f87e1f74b..028a8cdd10f 100644
--- a/src/metabase/sync/util.clj
+++ b/src/metabase/sync/util.clj
@@ -564,6 +564,7 @@
[:semantic-version {:optional true} [:or
[:sequential :int]
[:map {:closed true} [:major :int] [:minor :int]]]]
+ [:single-node {:optional true} :boolean]
[:tables-classified {:optional true} :int]
[:throwable {:optional true} [:maybe (ms/InstanceOfClass Throwable)]]
[:timezone-id {:optional true} [:maybe :string]]
diff --git a/src/metabase/task_history/schema.clj b/src/metabase/task_history/schema.clj
index f4472cada6b..fb3e4c90d14 100644
--- a/src/metabase/task_history/schema.clj
+++ b/src/metabase/task_history/schema.clj
@@ -107,7 +107,8 @@
[:semantic-version {:optional true} [:or
[:sequential :int]
[:map {:closed true} [:major :int] [:minor :int]]]]
- [:cloud {:optional true} :boolean]])
+ [:cloud {:optional true} :boolean]
+ [:single-node {:optional true} :boolean]])
(mr/def ::task-details.empty
[:map {:closed true}])
diff --git a/src/metabase/warehouses/schema.clj b/src/metabase/warehouses/schema.clj
index 98704a7da9d..865665abbcc 100644
--- a/src/metabase/warehouses/schema.clj
+++ b/src/metabase/warehouses/schema.clj
@@ -27,7 +27,8 @@
[:semantic-version {:optional true} [:or
[:sequential :int]
[:map {:closed true} [:major :int] [:minor :int]]]]
- [:cloud {:optional true} :boolean]])
+ [:cloud {:optional true} :boolean]
+ [:single-node {:optional true} :boolean]])
(mr/def ::database.write-data-details
"The `:write_data_details` column of a Database, decoded."
diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.clj
index 6e2d8e00295..a225482ec5d 100644
--- a/test/metabase/queries_rest/api/card_test.clj
+++ b/test/metabase/queries_rest/api/card_test.clj
@@ -4309,8 +4309,8 @@
[request]
(mt/test-drivers (mt/normal-drivers-with-feature :uploads)
(mt/with-discard-model-updates! [:model/Database] ; to restore any existing metabase_database.uploads_enabled=true
- (mt/with-temp [:model/Database {db-id :id} {:engine driver/*driver*}
- :model/Database {other-db-id :id} {:engine driver/*driver* :uploads_enabled true}
+ (mt/with-temp [:model/Database {db-id :id} {:engine driver/*driver* :dbms_version (:dbms_version (mt/db))}
+ :model/Database {other-db-id :id} {:engine driver/*driver* :dbms_version (:dbms_version (mt/db)) :uploads_enabled true}
:model/Table {table-id :id} {:db_id db-id, :is_upload true}
:model/Collection {collection-id :id} {}]
(let [card-defaults {:collection_id collection-id
diff --git a/test/metabase/test/data/impl.clj b/test/metabase/test/data/impl.clj
index 27ee52f6139..2ea4b7215f5 100644
--- a/test/metabase/test/data/impl.clj
+++ b/test/metabase/test/data/impl.clj
@@ -403,7 +403,7 @@
from the standard test database, and syncs it."
[f]
(let [{old-db-id :id, :as old-db} (*db-fn*)
- original-db (-> old-db copy-secrets (select-keys [:details :engine :name :settings]))
+ original-db (-> old-db copy-secrets (select-keys [:details :engine :name :settings :dbms_version]))
{new-db-id :id, :as new-db} (first (t2/insert-returning-instances! :model/Database original-db))]
(try
(copy-db-tables-and-fields! old-db-id new-db-id)
diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.clj
index e1aaa36001d..229ba84062d 100644
--- a/test/metabase/upload/impl_test.clj
+++ b/test/metabase/upload/impl_test.clj
@@ -1313,14 +1313,14 @@
java.lang.Exception
#"^Uploads are not enabled\.$"
(do-with-uploaded-example-csv!
- {:uploads-enabled false :schema-name "public", :table-prefix "uploaded_magic_"}
+ {:uploads-enabled false :table-prefix "uploaded_magic_"}
identity))))
(testing "Database ID must be valid"
(is (thrown-with-msg?
java.lang.Exception
#"^The uploads database does not exist\.$"
(do-with-uploaded-example-csv!
- {:db-id Integer/MAX_VALUE, :schema-name "public", :table-prefix "uploaded_magic_"}
+ {:db-id Integer/MAX_VALUE, :table-prefix "uploaded_magic_"}
identity))))
(testing "Uploads must be supported"
(mt/with-dynamic-fn-redefs [driver.u/supports? (constantly false)]
@@ -1328,7 +1328,7 @@
java.lang.Exception
#"^Uploads are not supported on [\w-]+ databases\."
(do-with-uploaded-example-csv!
- {:schema-name "public", :table-prefix "uploaded_magic_"}
+ {:table-prefix "uploaded_magic_"}
identity)))))
(testing "User must have write permissions on the collection"
(mt/with-non-admin-groups-no-root-collection-perms
@@ -1336,7 +1336,7 @@
java.lang.Exception
#"^You don't have permissions to do that\.$"
(do-with-uploaded-example-csv!
- {:user-id (mt/user->id :lucky) :schema-name "public", :table-prefix "uploaded_magic_"}
+ {:user-id (mt/user->id :lucky) :table-prefix "uploaded_magic_"}
identity)))))
(testing "File type must be allowed"
(is (thrown-with-msg?