Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Move dev-scripts to mage (#74032)

Riley Thompson24efe364b46e
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index bacbbb35dc7..610fd7538d5 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -263,6 +263,7 @@ jobs:
with:
compose-file: "modules/drivers/clickhouse/docker-compose.yml"
down-flags: "--volumes"
+ up-flags: "--pull always"
services: |
clickhouse
clickhouse_tls
diff --git a/.github/workflows/transforms-python-stress-test.yml b/.github/workflows/transforms-python-stress-test.yml
index 46242c1faf3..31d3fe69a0e 100644
--- a/.github/workflows/transforms-python-stress-test.yml
+++ b/.github/workflows/transforms-python-stress-test.yml
@@ -108,6 +108,7 @@ jobs:
with:
compose-file: "modules/drivers/clickhouse/docker-compose.yml"
down-flags: "--volumes"
+ up-flags: "--pull always"
services: |
clickhouse
clickhouse_tls
diff --git a/bb.edn b/bb.edn
index 27f0de4f69a..0a059be7984 100644
--- a/bb.edn
+++ b/bb.edn
@@ -132,19 +132,25 @@
start-db
{:doc "Start a db on a default port in docker"
:examples [["./bin/mage start-db postgres latest" "start the latest postgres db we support"]
- ["./bin/mage start-db mysql oldest" "start the oldest mysql db we support"]]
+ ["./bin/mage start-db mysql oldest" "start the oldest mysql db we support"]
+ ["./bin/mage start-db postgres latest --port 5555" "start postgres on a custom port"]]
:requires [[mage.start-db :as start-db]]
:arg-schema [:tuple
- [:enum {:name "db"} :postgres :mysql :mariadb :mongo]
- [:enum {:name "relative-version"} :oldest :latest]]
- :db-info {:postgres {:ports {:oldest 5432 :latest 5433} :eol-url "https://endoflife.date/api/postgres.json"}
- :mysql {:ports {:oldest 3308 :latest 3309} :eol-url "https://endoflife.date/api/mysql.json"}
- :mariadb {:ports {:oldest 3306 :latest 3307} :eol-url "https://endoflife.date/api/mariadb.json"}
- :mongo {:ports {:oldest 27017 :latest 27018} :eol-url "https://endoflife.date/api/mongodb.json"}}
+ [:enum :postgres :mysql :mariadb :mongo :clickhouse :sqlserver :oracle]
+ [:enum :oldest :latest]]
+ :options [["-p" "--port PORT" "Override the default host port for the container" :parse-fn Integer/parseInt]]
+ :db-info {:postgres {:ports {:oldest 5432 :latest 5433} :eol-url "https://endoflife.date/api/postgres.json"}
+ :mysql {:ports {:oldest 3308 :latest 3309} :eol-url "https://endoflife.date/api/mysql.json"}
+ :mariadb {:ports {:oldest 3306 :latest 3307} :eol-url "https://endoflife.date/api/mariadb.json"}
+ :mongo {:ports {:oldest 27017 :latest 27018} :eol-url "https://endoflife.date/api/mongodb.json"}
+ :sqlserver {:ports {:oldest 1433 :latest 1434} :eol-url "https://endoflife.date/api/mssqlserver.json"}
+ :oracle {:ports {:oldest 1521 :latest 1522} :eol-url "https://endoflife.date/api/oracle-database.json"}
+ ;; some databases aren't on endoflife.date
+ :clickhouse {:ports {:oldest 8124 :latest 8123} :eol-url nil}}
:usage-fn start-db/usage
:task (task!
(let [[db version] (:arguments parsed)]
- (start-db/start-db (:db-info (current-task)) db version)))}
+ (start-db/start-db (-> (current-task) :db-info db) db version (:options parsed))))}
nrepl
{:doc "Starts the babashka nrepl: helpful for mage development"
diff --git a/mage/src/mage/start_db.clj b/mage/src/mage/start_db.clj
index 16e3c009e18..29036587e98 100644
--- a/mage/src/mage/start_db.clj
+++ b/mage/src/mage/start_db.clj
@@ -17,32 +17,66 @@
(defn- ->deps-edn-alias [db version] (c/green ":db/" (name db) "-" (name version)))
-(defn- fetch-supported-versions [db-info database]
- (let [now (java.time.LocalDate/now)
- all-versions (-> (http/get (get-in db-info [database :eol-url])) :body (json/parse-string true))
- _ (u/debug "all-versions: \n" (with-out-str (t/table all-versions)))
- supported (->> all-versions
- (mapv (fn [m]
- (-> m
- (update :releaseDate #(and % (-> % java.time.LocalDate/parse)))
- (update :eol #(and % (-> % java.time.LocalDate/parse))))))
- (filter (fn [{:keys [^java.time.LocalDate eol]}]
- (and eol (.isAfter eol now))))
- (sort-by :releaseDate)
- vec)]
- (u/debug "supported: \n" (with-out-str (t/table supported)))
- supported))
-
-(defn- fetch-oldest-supported-version [db-info database]
- (let [versions (fetch-supported-versions db-info database)
- oldest-version (-> versions first :cycle)]
- (u/debug "OLDEST VERSION:" oldest-version)
- oldest-version))
+(defn all-versions [db-info] (-> db-info :eol-url http/get :body (json/parse-string true)))
+
+(defmulti ^:private fetch-oldest-supported-version
+ {:arglists '([database db-info])}
+ (fn [database db-info] database))
+
+(defn- fetch-oldest-supported-version* [db-info filter-fn sort-fn version-key]
+ (->> (all-versions db-info)
+ (map (fn [version] (update version :eol #(java.time.LocalDate/parse %))))
+ (filter (fn [{:keys [^java.time.LocalDate eol]}] (.isAfter eol (java.time.LocalDate/now))))
+ (filter filter-fn)
+ (sort-by sort-fn)
+ first
+ version-key))
+
+(defmethod fetch-oldest-supported-version :default
+ [_database db-info]
+ (fetch-oldest-supported-version*
+ db-info
+ (constantly true)
+ (fn [{:keys [releaseDate]}] (java.time.LocalDate/parse releaseDate))
+ :cycle))
+
+(defmethod fetch-oldest-supported-version :oracle
+ [_database db-info]
+ (fetch-oldest-supported-version*
+ db-info
+ (fn [{:keys [cycle]}] (and cycle (> (Integer/parseInt cycle) 19)))
+ (fn [{:keys [releaseDate]}] (java.time.LocalDate/parse releaseDate))
+ :cycle))
+
+(defmethod fetch-oldest-supported-version :sqlserver
+ [_database db-info]
+ (let [eol-version (fetch-oldest-supported-version*
+ db-info
+ (fn [{:keys [releaseLabel]}] (and releaseLabel (re-matches #"^\d{4} '__CODENAME__'$" releaseLabel)))
+ :releaseLabel
+ :releaseLabel)]
+ (str/replace eol-version #" '__CODENAME__'" "-latest")))
+
+(defmethod fetch-oldest-supported-version :clickhouse
+ [_database _db-info]
+ "23.3")
+
+(defmulti ^:private fetch-latest-supported-version
+ {:arglists '([database db-info])}
+ (fn [database db-info] database))
+
+(defmethod fetch-latest-supported-version :default
+ [_database _db-info]
+ "latest")
+
+(defmethod fetch-latest-supported-version :sqlserver
+ [_database _db-info]
+ "2025-latest")
(defn- resolve-version [db-info database version]
(if (= version :oldest)
- (fetch-oldest-supported-version db-info database)
- (cond-> version (keyword? version) name)))
+ (fetch-oldest-supported-version database db-info)
+ (fetch-latest-supported-version database db-info)))
;; Docker stuff:
@@ -57,8 +91,7 @@
(defmethod docker-cmd :postgres
[_db container-name resolved-version port]
- ["docker" "run"
- "-d"
+ ["docker" "run" "-d"
"-p" (str port ":5432")
;; "--network" "psql-metabase-network"
"-e" "POSTGRES_USER=metabase"
@@ -71,8 +104,7 @@
(defmethod docker-cmd :mysql
[_db container-name resolved-version port]
- ["docker" "run"
- "-d"
+ ["docker" "run" "-d"
"-p" (str port ":3306")
"-e" "MYSQL_DATABASE=metabase_test"
"-e" "MYSQL_ALLOW_EMPTY_PASSWORD=yes"
@@ -81,8 +113,7 @@
(defmethod docker-cmd :mariadb
[_db container-name resolved-version port]
- ["docker" "run"
- "-d"
+ ["docker" "run" "-d"
"-p" (str port ":3306")
"-e" "MYSQL_DATABASE=metabase_test"
"-e" "MYSQL_ALLOW_EMPTY_PASSWORD=yes"
@@ -91,14 +122,41 @@
(defmethod docker-cmd :mongo
[_db container-name resolved-version port]
- ["docker" "run"
- "-d"
+ ["docker" "run" "-d"
"-e" "MONGO_INITDB_ROOT_USERNAME=metabase"
"-e" "MONGO_INITDB_ROOT_PASSWORD=metasample123"
"-p" (str port ":27017")
"--name" container-name
(str "mongo:" resolved-version)])
+(defmethod docker-cmd :clickhouse
+ [_db container-name resolved-version port]
+ ["docker" "compose"
+ "-f" "modules/drivers/clickhouse/docker-compose.yml"
+ "up" "-d"
+ (if (= resolved-version "latest")
+ "clickhouse"
+ "clickhouse_older_version")]) ;; these are defined in modules/drivers/clickhouse/docker-compose.yml
+
+(defmethod docker-cmd :sqlserver
+ [_db container-name resolved-version port]
+ ["docker" "run" "-d"
+ "-p" (str port ":1433")
+ "-e" "ACCEPT_EULA=Y"
+ "-e" "SA_PASSWORD=P@ssw0rd"
+ "--name" container-name
+ (str "mcr.microsoft.com/mssql/server:" resolved-version)])
+
+(defmethod docker-cmd :oracle
+ [_db container-name resolved-version port]
+ ["docker" "run" "-d"
+ "-p" (str port ":1521")
+ "-e" "ORACLE_PASSWORD=password"
+ "--name" container-name
+ (if (= resolved-version "latest")
+ "gvenzl/oracle-free:latest"
+ (str "gvenzl/oracle-xe:" resolved-version))])
+
(defn- app-db? [db]
(contains? #{:postgres :mysql :mariadb} db))
@@ -135,21 +193,12 @@
(defn start-db
"Starts a db type + version in docker."
- [db-info db version]
- (let [port (get-in db-info [db :ports version])
- db (keyword db)
- version (cond-> version (string? version) keyword)
- resolved-version (resolve-version db-info db version)]
- (u/debug "PORT:" port)
- (cond (not (contains? db-info db))
- (do
- (println (c/red "Invalid DB: " (name db)))
- (println (usage {:db-info db-info})))
-
- (not (integer? port))
- (do
- (println (c/red "No port found for DB: " (name db) ", version: " (name version) ". See :db-info in bb.edn"))
- (println (usage {:db-info db-info})))
-
- :else
- (start-db! db version resolved-version port))))
+ ([db-info db version]
+ (start-db db-info db version nil))
+ ([db-info db version opts]
+ (when (and (= db :clickhouse) (:port opts))
+ (throw (ex-info (c/red "--port is not supported for clickhouse. Ports are configured in modules/drivers/clickhouse/docker-compose.yml") {:babashka/exit 1})))
+ (let [default-port (get-in db-info [:ports version])
+ port (or (:port opts) default-port)
+ resolved-version (resolve-version db-info db version)]
+ (start-db! db version resolved-version port))))
diff --git a/modules/drivers/clickhouse/docker-compose.yml b/modules/drivers/clickhouse/docker-compose.yml
index 6a2fbdb908d..7869a4d0582 100644
--- a/modules/drivers/clickhouse/docker-compose.yml
+++ b/modules/drivers/clickhouse/docker-compose.yml
@@ -5,8 +5,8 @@ services:
##########################################################################################################
clickhouse:
- image: 'clickhouse/clickhouse-server:25.2-alpine'
- container_name: 'metabase-driver-clickhouse-server'
+ image: 'clickhouse/clickhouse-server:latest-alpine'
+ container_name: 'mb-clickhouse-latest'
hostname: clickhouse
ports:
- '8123:8123'
@@ -51,8 +51,9 @@ services:
##########################################################################################################
clickhouse_older_version:
+ # if the oldest supported veresion changes please update fetch-oldest-supported-version in mage/mage/start_db.clj
image: 'clickhouse/clickhouse-server:23.3-alpine'
- container_name: 'metabase-driver-clickhouse-server-older-version'
+ container_name: 'mb-clickhouse-oldest'
hostname: clickhouse.older
ports:
- '8124:8123'
@@ -73,7 +74,7 @@ services:
##########################################################################################################
clickhouse_cluster_node1:
- image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}'
+ image: 'clickhouse/clickhouse-server:latest-alpine'
ulimits:
nofile:
soft: 262144
@@ -92,7 +93,7 @@ services:
- './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml'
clickhouse_cluster_node2:
- image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-25.2-alpine}'
+ image: 'clickhouse/clickhouse-server:latest-alpine'
ulimits:
nofile:
soft: 262144
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj
index eb80244e84e..d84c1f7344f 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj
@@ -58,7 +58,7 @@
(defn- set-role-throws-test!
[details-map]
(testing "throws when assigning a non-existent role"
- (is (thrown-with-msg? SQLException #"There is no role `asdf` in user directories."
+ (is (thrown-with-msg? SQLException #"There is no role `asdf` in `user directories`."
(sql-jdbc.execute/do-with-connection-with-options
:clickhouse (sql-jdbc.conn/connection-details->spec :clickhouse details-map) nil
(fn [^java.sql.Connection conn]
@@ -70,7 +70,7 @@
[:model/Database db {:engine :clickhouse :details details}]
(qp.store/with-metadata-provider (u/the-id db) (thunk db))))
-(deftest clickhouse-set-role
+(deftest clickhouse-set-role-test
(mt/test-driver :clickhouse
(let [user-details {:user "metabase_test_user"}
;; See docker-compose.yml for the port mappings