The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
CSV uploads for non-cloud clickhouse (#83878)
Riley Thompson02d40a0aabea
diff --git a/docs/databases/connections/clickhouse.md b/docs/databases/connections/clickhouse.mdindex 1127a0222c2..20310ede5a3 100644--- a/docs/databases/connections/clickhouse.md+++ b/docs/databases/connections/clickhouse.md@@ -105,6 +105,12 @@ A fingerprinting query examines the first 10,000 rows from each column and usesSet up an additional connection used for write operations. See [Writable connections](../writable-connection.md).+## Uploads++Metabase supports [CSV uploads](../uploads.md) to ClickHouse Cloud and self-hosted single-node ClickHouse deployments. Self-hosted clusters aren't supported.++For self-hosted deployments, the database user must be able to read the `system.clusters` table, as well as create tables and insert data.+## Model featuresThere aren't (yet) any model features for ClickHouse.diff --git a/docs/databases/uploads.md b/docs/databases/uploads.mdindex 3c08620fa08..cb0ecd15a2f 100644--- a/docs/databases/uploads.md+++ b/docs/databases/uploads.md@@ -21,7 +21,7 @@ To manage upload settings, admins can hit cmd/ctrl + K and search for "Settings- [MySQL](../databases/connections/mysql.md)- [Snowflake](../databases/connections/snowflake.md)- [Redshift](../databases/connections/redshift.md)-- [ClickHouse](../databases/connections/clickhouse.md) (only supported on ClickHouse Cloud)+- [ClickHouse](../databases/connections/clickhouse.md#uploads) (ClickHouse Cloud and self-hosted single-node deployments)## Setting up uploadsdiff --git a/enterprise/backend/test/metabase_enterprise/sandbox/upload_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/upload_test.cljindex 2baf6262ca0..aeee258c378 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/upload_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/upload_test.clj@@ -43,7 +43,7 @@(upload-test/with-uploads-enabled!(mt/dataset dataset(mt/with-temp [:model/Collection collection {}- :model/Database {db-id :id} {:engine driver/*driver* :details (:details (mt/db))}+ :model/Database {db-id :id} {:engine driver/*driver* :details (:details (mt/db)) :dbms_version (:dbms_version (mt/db))}:model/Table {table-id :id} {:db_id db-id:is_upload true}:model/Card {card-id :iddiff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.cljindex 15315877cd1..1da981ecd48 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj@@ -3,6 +3,7 @@(:require[clojure.java.jdbc :as jdbc][clojure.string :as str]+ [honey.sql :as sql][metabase.driver :as driver][metabase.driver-api.core :as driver-api][metabase.driver.clickhouse-introspection]@@ -16,6 +17,8 @@[metabase.driver.sql-jdbc.common :as sql-jdbc.common][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]+ [metabase.driver.sql-jdbc.quoting :refer [dot-qualified quote-identifier with-quoting]]+ [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.util :as sql.u][metabase.util :as u]@@ -186,7 +189,8 @@(sql-jdbc.common/handle-additional-options details :separator-style :url))))(defmethod driver/database-supports? [:clickhouse :uploads] [_driver _feature db]- (boolean (-> db clickhouse-version/dbms-version :cloud)))+ (let [{:keys [cloud single-node]} (clickhouse-version/dbms-version db)]+ (boolean (or cloud single-node))))(defmethod driver/can-connect? :clickhouse[driver details]@@ -254,6 +258,20 @@:metabase.upload/datetime "Nullable(DateTime64(3))":metabase.upload/offset-datetime nil))+;; ClickHouse changes a column's type with MODIFY COLUMN; the default ALTER COLUMN is a syntax error.+(defmethod sql-jdbc.sync/alter-table-columns-sql :clickhouse+ [driver table-name column-definitions & _opts]+ (with-quoting driver+ (first (sql/format {:alter-table (dot-qualified table-name)+ :modify-column (map (fn [[column-name type-and-constraints]]+ (into [(quote-identifier column-name)]+ (if (string? type-and-constraints)+ [[:raw type-and-constraints]]+ type-and-constraints)))+ column-definitions)}+ :quoted true+ :dialect (sql.qp/quote-style driver)))))+(defmulti ^:private type->database-type"Internal type->database-type multimethod for ClickHouse that dispatches on type."{:arglists '([type])}@@ -278,9 +296,10 @@(defmethod driver/table-name-length-limit :clickhouse[_driver]- ;; FIXME: This is a lie because you're really limited by a filesystems' limits, because Clickhouse uses- ;; filenames as table/column names. But its an approximation- 206)+ ;; ClickHouse stores table metadata as files, so the real limit is on the combined length of the database and table+ ;; names: on recent versions `max table name length = 213 - (length of the database name)`, so the old value of 206+ ;; only held for a 7-character database such as `default`. 128 leaves room for database names up to 85 characters.+ 128)(defn- escape-ident;; Backslash-escape rather than double the backtick: ClickHouse identifiers follow string-literal escaping, wherediff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.cljindex 20bd4b86111..9513ce58bb3 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_qp.clj@@ -584,7 +584,7 @@(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/BIGINT][_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i](fn []- (with-null-check rs (.getBigDecimal rs i))))+ (with-null-check rs (.getLong rs i))))(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/INTEGER][_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i]diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.cljindex 1677b19ceaa..b2ed976c1f1 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse_version.clj@@ -8,7 +8,10 @@[metabase.driver.connection :as driver.conn][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]- [metabase.driver.util :as driver.u]))+ [metabase.driver.util :as driver.u]+ [metabase.util.log :as log])+ (:import+ (java.sql Connection SQLException)))(set! *warn-on-reflection* true)@@ -20,6 +23,18 @@(str "WITH s AS (SELECT version() AS ver, splitByChar('.', ver) AS verSplit) ""SELECT s.ver, toInt32(verSplit[1]), toInt32(verSplit[2]) FROM s"))+(defn- single-node?+ "Return true if system.clusters has no entries marked as non-local.+ This check uses configuration data and does not verify the number of active nodes."+ [^Connection conn]+ (try+ (with-open [stmt (.createStatement conn)+ rset (.executeQuery stmt "SELECT count() = 0 FROM system.clusters WHERE is_local = 0")]+ (and (.next rset) (.getBoolean rset 1)))+ (catch SQLException e+ (log/warn e "Could not determine whether ClickHouse is a single-node deployment.")+ false)))+(def ^:private ^{:arglists '([database])} get-clickhouse-version(memoize/ttl^{:clojure.core.memoize/args-fn (fn [[database]] (driver.conn/effective-details database))}@@ -33,14 +48,15 @@ver-rset (.executeQuery ver-stmt clickhouse-version-query)cloud-stmt (.createStatement conn)cloud-rset (.executeQuery cloud-stmt "SELECT value='1' FROM system.settings WHERE name='cloud_mode'")]- (cond-> nil- (.next ver-rset)- (assoc :version (.getString ver-rset 1)- :semantic-version {:major (.getInt ver-rset 2)- :minor (.getInt ver-rset 3)})+ (let [cloud? (and (.next cloud-rset) (.getBoolean cloud-rset 1))]+ (cond-> {:cloud cloud?}+ (.next ver-rset)+ (assoc :version (.getString ver-rset 1)+ :semantic-version {:major (.getInt ver-rset 2)+ :minor (.getInt ver-rset 3)})- (.next cloud-rset)- (assoc :cloud (.getBoolean cloud-rset 1)))))))+ (not cloud?)+ (assoc :single-node (single-node? conn)))))))):ttl/threshold default-cache-ttl))(defmethod driver/dbms-version :clickhousediff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.cljindex 16b3a2c6c7a..28e53e28c85 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj@@ -8,7 +8,6 @@[metabase.driver :as driver][metabase.driver.clickhouse :as clickhouse][metabase.driver.clickhouse-qp :as clickhouse-qp]- [metabase.driver.clickhouse-version :as clickhouse-version][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc :as sql-jdbc][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]@@ -728,54 +727,53 @@(deftest ^:parallel uploads-supported-test(mt/test-driver :clickhouse- (is (false? (driver/database-supports? driver/*driver* :uploads (mt/db))))+ (is (true? (driver/database-supports? driver/*driver* :uploads (mt/db))))(is (true? (driver/database-supports? driver/*driver* :uploads (assoc-in (mt/db) [:dbms-version :cloud] true))))(is (true? (driver/database-supports? driver/*driver* :uploads (assoc-in (mt/db) [:dbms_version :cloud] true))))))(deftest ^:synchronized csv-upload-and-sync-test- (testing "ClickHouse CSV uploads work correctly when cloud mode is enabled"+ (testing "ClickHouse CSV uploads work correctly"(mt/test-driver :clickhouse- (mt/with-dynamic-fn-redefs [clickhouse-version/dbms-version (constantly {:cloud true- :version "24.8.1"- :semantic-version {:major 24 :minor 8}})]- (let [details (-> (mt/dbdef->connection-details :clickhouse :db {:database-name "uploads_schema"})- (assoc :enable-multiple-db false))- conn-spec (sql-jdbc.conn/connection-details->spec :clickhouse details)]- (driver/create-schema-if-needed! :clickhouse conn-spec "uploads_schema")- (try- (mt/with-temp [:model/Database db {:engine :clickhouse- :details details}]- (is (true? (driver/database-supports? :clickhouse :uploads db)))- (testing "an upload schema is required"- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"A schema has not been set."- (upload-test/do-with-uploaded-example-csv!- {:db-id (:id db)- :auxiliary-sync-steps :synchronous- :schema-name ""}- identity))))- (testing "upload models work after sync"- (upload-test/do-with-uploaded-example-csv!- {:db-id (:id db)- :auxiliary-sync-steps :synchronous- :schema-name "uploads_schema"}- (fn [model]- (let [query-model (fn []- (let [mp (lib-be/application-database-metadata-provider (:id db))- card (lib.metadata/card mp (:id model))]- (->> (lib/query mp card)- (qp/process-query)- (mt/formatted-rows [int str]))))]- (is (= [[1 " Luke Skywalker"]- [2 " Darth Vader"]]- (query-model)))- (sync/sync-database! db {:scan :schema})- (is (= [[1 " Luke Skywalker"]- [2 " Darth Vader"]]- (query-model))))))))- (finally- (jdbc/execute! conn-spec ["DROP DATABASE IF EXISTS `uploads_schema`"]))))))))+ (let [details (-> (mt/dbdef->connection-details :clickhouse :db {:database-name "uploads_schema"})+ (assoc :enable-multiple-db false))+ dbms-version (driver/dbms-version :clickhouse (mt/db))+ conn-spec (sql-jdbc.conn/connection-details->spec :clickhouse details)]+ (driver/create-schema-if-needed! :clickhouse conn-spec "uploads_schema")+ (try+ (mt/with-temp [:model/Database db {:engine :clickhouse+ :details details+ :dbms_version dbms-version}]+ (is (true? (driver/database-supports? :clickhouse :uploads db)))+ (testing "an upload schema is required"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"A schema has not been set."+ (upload-test/do-with-uploaded-example-csv!+ {:db-id (:id db)+ :auxiliary-sync-steps :synchronous+ :schema-name ""}+ identity))))+ (testing "upload models work after sync"+ (upload-test/do-with-uploaded-example-csv!+ {:db-id (:id db)+ :auxiliary-sync-steps :synchronous+ :schema-name "uploads_schema"}+ (fn [model]+ (let [query-model (fn []+ (let [mp (lib-be/application-database-metadata-provider (:id db))+ card (lib.metadata/card mp (:id model))]+ (->> (lib/query mp card)+ (qp/process-query)+ (mt/formatted-rows [int str]))))]+ (is (= [[1 " Luke Skywalker"]+ [2 " Darth Vader"]]+ (query-model)))+ (sync/sync-database! db {:scan :schema})+ (is (= [[1 " Luke Skywalker"]+ [2 " Darth Vader"]]+ (query-model))))))))+ (finally+ (jdbc/execute! conn-spec ["DROP DATABASE IF EXISTS `uploads_schema`"])))))))(deftest ^:parallel type->database-type-test(testing "type->database-type multimethod returns correct ClickHouse types"diff --git a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.cljindex df6097df561..e28bb96cb09 100644--- a/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj+++ b/modules/drivers/clickhouse/test/metabase/test/data/clickhouse.clj@@ -191,7 +191,7 @@(defmethod sql.tx/add-fk-sql :clickhouse [& _] nil)-(defmethod sql.tx/session-schema :clickhouse [_] "default")+(defmethod sql.tx/session-schema :clickhouse [_] (-> (mt/db) :details :db))(defmethod sql.tx/generated-column-sql :clickhouse [_ expr](format "ALIAS (%s)" expr))diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex 88daf475d98..0b17dc7bbd9 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -1095,7 +1095,8 @@[:semantic-version {:optional true} [:maybe [:or[:sequential :int][:map {:closed true} [:major :int] [:minor :int]]]]]- [:cloud {:optional true} [:maybe :boolean]]]]]+ [:cloud {:optional true} [:maybe :boolean]]+ [:single-node {:optional true} [:maybe :boolean]]]]][:details {:optional true} ::lib.schema.common/database-details][:engine {:optional true} [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]][:features {:optional true} [:set [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]]]diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.cljindex 05f87e1f74b..028a8cdd10f 100644--- a/src/metabase/sync/util.clj+++ b/src/metabase/sync/util.clj@@ -564,6 +564,7 @@[:semantic-version {:optional true} [:or[:sequential :int][:map {:closed true} [:major :int] [:minor :int]]]]+ [:single-node {:optional true} :boolean][:tables-classified {:optional true} :int][:throwable {:optional true} [:maybe (ms/InstanceOfClass Throwable)]][:timezone-id {:optional true} [:maybe :string]]diff --git a/src/metabase/task_history/schema.clj b/src/metabase/task_history/schema.cljindex f4472cada6b..fb3e4c90d14 100644--- a/src/metabase/task_history/schema.clj+++ b/src/metabase/task_history/schema.clj@@ -107,7 +107,8 @@[:semantic-version {:optional true} [:or[:sequential :int][:map {:closed true} [:major :int] [:minor :int]]]]- [:cloud {:optional true} :boolean]])+ [:cloud {:optional true} :boolean]+ [:single-node {:optional true} :boolean]])(mr/def ::task-details.empty[:map {:closed true}])diff --git a/src/metabase/warehouses/schema.clj b/src/metabase/warehouses/schema.cljindex 98704a7da9d..865665abbcc 100644--- a/src/metabase/warehouses/schema.clj+++ b/src/metabase/warehouses/schema.clj@@ -27,7 +27,8 @@[:semantic-version {:optional true} [:or[:sequential :int][:map {:closed true} [:major :int] [:minor :int]]]]- [:cloud {:optional true} :boolean]])+ [:cloud {:optional true} :boolean]+ [:single-node {:optional true} :boolean]])(mr/def ::database.write-data-details"The `:write_data_details` column of a Database, decoded."diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.cljindex 6e2d8e00295..a225482ec5d 100644--- a/test/metabase/queries_rest/api/card_test.clj+++ b/test/metabase/queries_rest/api/card_test.clj@@ -4309,8 +4309,8 @@[request](mt/test-drivers (mt/normal-drivers-with-feature :uploads)(mt/with-discard-model-updates! [:model/Database] ; to restore any existing metabase_database.uploads_enabled=true- (mt/with-temp [:model/Database {db-id :id} {:engine driver/*driver*}- :model/Database {other-db-id :id} {:engine driver/*driver* :uploads_enabled true}+ (mt/with-temp [:model/Database {db-id :id} {:engine driver/*driver* :dbms_version (:dbms_version (mt/db))}+ :model/Database {other-db-id :id} {:engine driver/*driver* :dbms_version (:dbms_version (mt/db)) :uploads_enabled true}:model/Table {table-id :id} {:db_id db-id, :is_upload true}:model/Collection {collection-id :id} {}](let [card-defaults {:collection_id collection-iddiff --git a/test/metabase/test/data/impl.clj b/test/metabase/test/data/impl.cljindex 27ee52f6139..2ea4b7215f5 100644--- a/test/metabase/test/data/impl.clj+++ b/test/metabase/test/data/impl.clj@@ -403,7 +403,7 @@from the standard test database, and syncs it."[f](let [{old-db-id :id, :as old-db} (*db-fn*)- original-db (-> old-db copy-secrets (select-keys [:details :engine :name :settings]))+ original-db (-> old-db copy-secrets (select-keys [:details :engine :name :settings :dbms_version])){new-db-id :id, :as new-db} (first (t2/insert-returning-instances! :model/Database original-db))](try(copy-db-tables-and-fields! old-db-id new-db-id)diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.cljindex e1aaa36001d..229ba84062d 100644--- a/test/metabase/upload/impl_test.clj+++ b/test/metabase/upload/impl_test.clj@@ -1313,14 +1313,14 @@java.lang.Exception#"^Uploads are not enabled\.$"(do-with-uploaded-example-csv!- {:uploads-enabled false :schema-name "public", :table-prefix "uploaded_magic_"}+ {:uploads-enabled false :table-prefix "uploaded_magic_"}identity))))(testing "Database ID must be valid"(is (thrown-with-msg?java.lang.Exception#"^The uploads database does not exist\.$"(do-with-uploaded-example-csv!- {:db-id Integer/MAX_VALUE, :schema-name "public", :table-prefix "uploaded_magic_"}+ {:db-id Integer/MAX_VALUE, :table-prefix "uploaded_magic_"}identity))))(testing "Uploads must be supported"(mt/with-dynamic-fn-redefs [driver.u/supports? (constantly false)]@@ -1328,7 +1328,7 @@java.lang.Exception#"^Uploads are not supported on [\w-]+ databases\."(do-with-uploaded-example-csv!- {:schema-name "public", :table-prefix "uploaded_magic_"}+ {:table-prefix "uploaded_magic_"}identity)))))(testing "User must have write permissions on the collection"(mt/with-non-admin-groups-no-root-collection-perms@@ -1336,7 +1336,7 @@java.lang.Exception#"^You don't have permissions to do that\.$"(do-with-uploaded-example-csv!- {:user-id (mt/user->id :lucky) :schema-name "public", :table-prefix "uploaded_magic_"}+ {:user-id (mt/user->id :lucky) :table-prefix "uploaded_magic_"}identity)))))(testing "File type must be allowed"(is (thrown-with-msg?