The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Move dev-scripts to mage (#74032)
Riley Thompson24efe364b46e
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex bacbbb35dc7..610fd7538d5 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -263,6 +263,7 @@ jobs:with:compose-file: "modules/drivers/clickhouse/docker-compose.yml"down-flags: "--volumes"+ up-flags: "--pull always"services: |clickhouseclickhouse_tlsdiff --git a/.github/workflows/transforms-python-stress-test.yml b/.github/workflows/transforms-python-stress-test.ymlindex 46242c1faf3..31d3fe69a0e 100644--- a/.github/workflows/transforms-python-stress-test.yml+++ b/.github/workflows/transforms-python-stress-test.yml@@ -108,6 +108,7 @@ jobs:with:compose-file: "modules/drivers/clickhouse/docker-compose.yml"down-flags: "--volumes"+ up-flags: "--pull always"services: |clickhouseclickhouse_tlsdiff --git a/bb.edn b/bb.ednindex 27f0de4f69a..0a059be7984 100644--- a/bb.edn+++ b/bb.edn@@ -132,19 +132,25 @@start-db{:doc "Start a db on a default port in docker":examples [["./bin/mage start-db postgres latest" "start the latest postgres db we support"]- ["./bin/mage start-db mysql oldest" "start the oldest mysql db we support"]]+ ["./bin/mage start-db mysql oldest" "start the oldest mysql db we support"]+ ["./bin/mage start-db postgres latest --port 5555" "start postgres on a custom port"]]:requires [[mage.start-db :as start-db]]:arg-schema [:tuple- [:enum {:name "db"} :postgres :mysql :mariadb :mongo]- [:enum {:name "relative-version"} :oldest :latest]]- :db-info {:postgres {:ports {:oldest 5432 :latest 5433} :eol-url "https://endoflife.date/api/postgres.json"}- :mysql {:ports {:oldest 3308 :latest 3309} :eol-url "https://endoflife.date/api/mysql.json"}- :mariadb {:ports {:oldest 3306 :latest 3307} :eol-url "https://endoflife.date/api/mariadb.json"}- :mongo {:ports {:oldest 27017 :latest 27018} :eol-url "https://endoflife.date/api/mongodb.json"}}+ [:enum :postgres :mysql :mariadb :mongo :clickhouse :sqlserver :oracle]+ [:enum :oldest :latest]]+ :options [["-p" "--port PORT" "Override the default host port for the container" :parse-fn Integer/parseInt]]+ :db-info {:postgres {:ports {:oldest 5432 :latest 5433} :eol-url "https://endoflife.date/api/postgres.json"}+ :mysql {:ports {:oldest 3308 :latest 3309} :eol-url "https://endoflife.date/api/mysql.json"}+ :mariadb {:ports {:oldest 3306 :latest 3307} :eol-url "https://endoflife.date/api/mariadb.json"}+ :mongo {:ports {:oldest 27017 :latest 27018} :eol-url "https://endoflife.date/api/mongodb.json"}+ :sqlserver {:ports {:oldest 1433 :latest 1434} :eol-url "https://endoflife.date/api/mssqlserver.json"}+ :oracle {:ports {:oldest 1521 :latest 1522} :eol-url "https://endoflife.date/api/oracle-database.json"}+ ;; some databases aren't on endoflife.date+ :clickhouse {:ports {:oldest 8124 :latest 8123} :eol-url nil}}:usage-fn start-db/usage:task (task!(let [[db version] (:arguments parsed)]- (start-db/start-db (:db-info (current-task)) db version)))}+ (start-db/start-db (-> (current-task) :db-info db) db version (:options parsed))))}nrepl{:doc "Starts the babashka nrepl: helpful for mage development"diff --git a/mage/src/mage/start_db.clj b/mage/src/mage/start_db.cljindex 16e3c009e18..29036587e98 100644--- a/mage/src/mage/start_db.clj+++ b/mage/src/mage/start_db.clj@@ -17,32 +17,66 @@(defn- ->deps-edn-alias [db version] (c/green ":db/" (name db) "-" (name version)))-(defn- fetch-supported-versions [db-info database]- (let [now (java.time.LocalDate/now)- all-versions (-> (http/get (get-in db-info [database :eol-url])) :body (json/parse-string true))- _ (u/debug "all-versions: \n" (with-out-str (t/table all-versions)))- supported (->> all-versions- (mapv (fn [m]- (-> m- (update :releaseDate #(and % (-> % java.time.LocalDate/parse)))- (update :eol #(and % (-> % java.time.LocalDate/parse))))))- (filter (fn [{:keys [^java.time.LocalDate eol]}]- (and eol (.isAfter eol now))))- (sort-by :releaseDate)- vec)]- (u/debug "supported: \n" (with-out-str (t/table supported)))- supported))--(defn- fetch-oldest-supported-version [db-info database]- (let [versions (fetch-supported-versions db-info database)- oldest-version (-> versions first :cycle)]- (u/debug "OLDEST VERSION:" oldest-version)- oldest-version))+(defn all-versions [db-info] (-> db-info :eol-url http/get :body (json/parse-string true)))++(defmulti ^:private fetch-oldest-supported-version+ {:arglists '([database db-info])}+ (fn [database db-info] database))++(defn- fetch-oldest-supported-version* [db-info filter-fn sort-fn version-key]+ (->> (all-versions db-info)+ (map (fn [version] (update version :eol #(java.time.LocalDate/parse %))))+ (filter (fn [{:keys [^java.time.LocalDate eol]}] (.isAfter eol (java.time.LocalDate/now))))+ (filter filter-fn)+ (sort-by sort-fn)+ first+ version-key))++(defmethod fetch-oldest-supported-version :default+ [_database db-info]+ (fetch-oldest-supported-version*+ db-info+ (constantly true)+ (fn [{:keys [releaseDate]}] (java.time.LocalDate/parse releaseDate))+ :cycle))++(defmethod fetch-oldest-supported-version :oracle+ [_database db-info]+ (fetch-oldest-supported-version*+ db-info+ (fn [{:keys [cycle]}] (and cycle (> (Integer/parseInt cycle) 19)))+ (fn [{:keys [releaseDate]}] (java.time.LocalDate/parse releaseDate))+ :cycle))++(defmethod fetch-oldest-supported-version :sqlserver+ [_database db-info]+ (let [eol-version (fetch-oldest-supported-version*+ db-info+ (fn [{:keys [releaseLabel]}] (and releaseLabel (re-matches #"^\d{4} '__CODENAME__'$" releaseLabel)))+ :releaseLabel+ :releaseLabel)]+ (str/replace eol-version #" '__CODENAME__'" "-latest")))++(defmethod fetch-oldest-supported-version :clickhouse+ [_database _db-info]+ "23.3")++(defmulti ^:private fetch-latest-supported-version+ {:arglists '([database db-info])}+ (fn [database db-info] database))++(defmethod fetch-latest-supported-version :default+ [_database _db-info]+ "latest")++(defmethod fetch-latest-supported-version :sqlserver+ [_database _db-info]+ "2025-latest")(defn- resolve-version [db-info database version](if (= version :oldest)- (fetch-oldest-supported-version db-info database)- (cond-> version (keyword? version) name)))+ (fetch-oldest-supported-version database db-info)+ (fetch-latest-supported-version database db-info)));; Docker stuff:@@ -57,8 +91,7 @@(defmethod docker-cmd :postgres[_db container-name resolved-version port]- ["docker" "run"- "-d"+ ["docker" "run" "-d""-p" (str port ":5432");; "--network" "psql-metabase-network""-e" "POSTGRES_USER=metabase"@@ -71,8 +104,7 @@(defmethod docker-cmd :mysql[_db container-name resolved-version port]- ["docker" "run"- "-d"+ ["docker" "run" "-d""-p" (str port ":3306")"-e" "MYSQL_DATABASE=metabase_test""-e" "MYSQL_ALLOW_EMPTY_PASSWORD=yes"@@ -81,8 +113,7 @@(defmethod docker-cmd :mariadb[_db container-name resolved-version port]- ["docker" "run"- "-d"+ ["docker" "run" "-d""-p" (str port ":3306")"-e" "MYSQL_DATABASE=metabase_test""-e" "MYSQL_ALLOW_EMPTY_PASSWORD=yes"@@ -91,14 +122,41 @@(defmethod docker-cmd :mongo[_db container-name resolved-version port]- ["docker" "run"- "-d"+ ["docker" "run" "-d""-e" "MONGO_INITDB_ROOT_USERNAME=metabase""-e" "MONGO_INITDB_ROOT_PASSWORD=metasample123""-p" (str port ":27017")"--name" container-name(str "mongo:" resolved-version)])+(defmethod docker-cmd :clickhouse+ [_db container-name resolved-version port]+ ["docker" "compose"+ "-f" "modules/drivers/clickhouse/docker-compose.yml"+ "up" "-d"+ (if (= resolved-version "latest")+ "clickhouse"+ "clickhouse_older_version")]) ;; these are defined in modules/drivers/clickhouse/docker-compose.yml++(defmethod docker-cmd :sqlserver+ [_db container-name resolved-version port]+ ["docker" "run" "-d"+ "-p" (str port ":1433")+ "-e" "ACCEPT_EULA=Y"+ "-e" "SA_PASSWORD=P@ssw0rd"+ "--name" container-name+ (str "mcr.microsoft.com/mssql/server:" resolved-version)])++(defmethod docker-cmd :oracle+ [_db container-name resolved-version port]+ ["docker" "run" "-d"+ "-p" (str port ":1521")+ "-e" "ORACLE_PASSWORD=password"+ "--name" container-name+ (if (= resolved-version "latest")+ "gvenzl/oracle-free:latest"+ (str "gvenzl/oracle-xe:" resolved-version))])+(defn- app-db? [db](contains? #{:postgres :mysql :mariadb} db))@@ -135,21 +193,12 @@(defn start-db"Starts a db type + version in docker."- [db-info db version]- (let [port (get-in db-info [db :ports version])- db (keyword db)- version (cond-> version (string? version) keyword)- resolved-version (resolve-version db-info db version)]- (u/debug "PORT:" port)- (cond (not (contains? db-info db))- (do- (println (c/red "Invalid DB: " (name db)))- (println (usage {:db-info db-info})))-- (not (integer? port))- (do- (println (c/red "No port found for DB: " (name db) ", version: " (name version) ". See :db-info in bb.edn"))- (println (usage {:db-info db-info})))-- :else- (start-db! db version resolved-version port))))+ ([db-info db version]+ (start-db db-info db version nil))+ ([db-info db version opts]+ (when (and (= db :clickhouse) (:port opts))+ (throw (ex-info (c/red "--port is not supported for clickhouse. Ports are configured in modules/drivers/clickhouse/docker-compose.yml") {:babashka/exit 1})))+ (let [default-port (get-in db-info [:ports version])+ port (or (:port opts) default-port)+ resolved-version (resolve-version db-info db version)]+ (start-db! db version resolved-version port))))diff --git a/modules/drivers/clickhouse/docker-compose.yml b/modules/drivers/clickhouse/docker-compose.ymlindex 6a2fbdb908d..7869a4d0582 100644--- a/modules/drivers/clickhouse/docker-compose.yml+++ b/modules/drivers/clickhouse/docker-compose.yml@@ -5,8 +5,8 @@ services:##########################################################################################################clickhouse:- image: 'clickhouse/clickhouse-server:25.2-alpine'- container_name: 'metabase-driver-clickhouse-server'+ image: 'clickhouse/clickhouse-server:latest-alpine'+ container_name: 'mb-clickhouse-latest'hostname: clickhouseports:- '8123:8123'@@ -51,8 +51,9 @@ services:##########################################################################################################clickhouse_older_version:+ # if the oldest supported veresion changes please update fetch-oldest-supported-version in mage/mage/start_db.cljimage: 'clickhouse/clickhouse-server:23.3-alpine'- container_name: 'metabase-driver-clickhouse-server-older-version'+ container_name: 'mb-clickhouse-oldest'hostname: clickhouse.olderports:- '8124:8123'@@ -73,7 +74,7 @@ services:##########################################################################################################clickhouse_cluster_node1:- image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}'+ image: 'clickhouse/clickhouse-server:latest-alpine'ulimits:nofile:soft: 262144@@ -92,7 +93,7 @@ services:- './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml'clickhouse_cluster_node2:- image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}'+ image: 'clickhouse/clickhouse-server:latest-alpine'ulimits:nofile:soft: 262144diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.cljindex eb80244e84e..d84c1f7344f 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj@@ -58,7 +58,7 @@(defn- set-role-throws-test nil(fn [^java.sql.Connection conn]@@ -70,7 +70,7 @@[:model/Database db {:engine :clickhouse :details details}](qp.store/with-metadata-provider (u/the-id db) (thunk db))))-(deftest clickhouse-set-role+(deftest clickhouse-set-role-test(mt/test-driver :clickhouse(let [user-details {:user "metabase_test_user"};; See docker-compose.yml for the port mappings