Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add AWS IAM Authentication Support for PostgreSQL and MySQL (#65418)

Nicola Momettoff099455d047
diff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.yml
index dfda83f3fff..c88368d2a76 100644
--- a/.github/workflows/app-db.yml
+++ b/.github/workflows/app-db.yml
@@ -137,11 +137,13 @@ jobs:
image: cimg/mysql:8.0
env:
enable-ssl-tests: 'false'
+ enable-aws-iam-tests: 'false'
- name: MySQL Latest
junit-name: be-tests-mysql-latest-ee
image: mysql:latest
env:
enable-ssl-tests: 'true'
+ enable-aws-iam-tests: 'true'
job:
- name: Enterprise Tests
build-static-viz: false
@@ -211,6 +213,14 @@ jobs:
MB_MYSQL_SSL_TEST_SSL_CERT: ${{ secrets.MB_MYSQL_SSL_TEST_SSL_CERT }}
MB_MYSQL_SSL_TEST_USER: metabase
MB_MYSQL_SSL_TEST_PASSWORD: ${{ secrets.MYSQL_RDS_SSL_INSTANCE_PASSWORD }}
+ MB_MYSQL_AWS_IAM_TEST: ${{ matrix.version.env.enable-aws-iam-tests }}
+ MB_MYSQL_AWS_IAM_TEST_HOST: ${{ secrets.MYSQL_AURORA_IAM_INSTANCE_HOST }}
+ MB_MYSQL_AWS_IAM_TEST_PORT: 3306
+ MB_MYSQL_AWS_IAM_TEST_USER: my_iam_user
+ MB_MYSQL_AWS_IAM_TEST_DBNAME: test_db
+ MB_MYSQL_AWS_IAM_TEST_SSL_CERT: trust
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AURORA_IAM_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AURORA_IAM_SECRET_ACCESS_KEY }}
# for historic reasons (I don't want to mess around with required jobs) the job name should be something like
# "be-tests-mariadb-10-2-ee (0)"
name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
@@ -249,11 +259,13 @@ jobs:
docker-image: postgres:12-alpine
env:
enable-ssl-tests: 'false'
+ enable-aws-iam-tests: 'false'
- name: Postgres Latest
junit-name: postgres-latest-ee
docker-image: postgres:latest
env:
enable-ssl-tests: 'true'
+ enable-aws-iam-tests: 'true'
job:
- name: Enterprise Tests
build-static-viz: false
@@ -305,6 +317,13 @@ jobs:
MB_POSTGRES_SSL_TEST_SSL: ${{ matrix.version.env.enable-ssl-tests }}
MB_POSTGRES_SSL_TEST_SSL_MODE: verify-full
MB_POSTGRES_SSL_TEST_SSL_ROOT_CERT_PATH: 'test-resources/certificates/us-east-2-bundle.pem'
+ MB_POSTGRES_AWS_IAM_TEST: ${{ matrix.version.env.enable-aws-iam-tests }}
+ MB_POSTGRES_AWS_IAM_TEST_HOST: ${{ secrets.POSTGRES_AURORA_IAM_INSTANCE_HOST }}
+ MB_POSTGRES_AWS_IAM_TEST_PORT: 5432
+ MB_POSTGRES_AWS_IAM_TEST_USER: my_iam_user
+ MB_POSTGRES_AWS_IAM_TEST_DBNAME: test_db
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AURORA_IAM_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AURORA_IAM_SECRET_ACCESS_KEY }}
services:
postgres:
image: ${{ matrix.version.docker-image }}
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 9acb6a88039..f4d29700768 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -423,21 +423,25 @@ jobs:
image: circleci/mariadb:10.2.23
env:
enable-ssl-tests: 'false'
+ enable-aws-iam-tests: 'false'
- name: MariaDB Latest
junit-name: be-tests-mariadb-latest-ee
image: circleci/mariadb:latest
env:
enable-ssl-tests: 'false'
+ enable-aws-iam-tests: 'false'
- name: MySQL 8.0
junit-name: be-tests-mysql-8-0-ee
image: mysql:8.0
env:
enable-ssl-tests: 'false'
+ enable-aws-iam-tests: 'false'
- name: MySQL Latest
junit-name: be-tests-mysql-latest-ee
image: mysql:latest
env:
enable-ssl-tests: 'true'
+ enable-aws-iam-tests: 'true'
job:
- name: Driver Tests
test-args: >-
@@ -496,6 +500,14 @@ jobs:
MB_MYSQL_SSL_TEST_SSL_CERT: ${{ secrets.MB_MYSQL_SSL_TEST_SSL_CERT }}
MB_MYSQL_SSL_TEST_USER: metabase
MB_MYSQL_SSL_TEST_PASSWORD: ${{ secrets.MYSQL_RDS_SSL_INSTANCE_PASSWORD }}
+ MB_MYSQL_AWS_IAM_TEST: ${{ matrix.version.env.enable-aws-iam-tests }}
+ MB_MYSQL_AWS_IAM_TEST_HOST: ${{ secrets.MYSQL_AURORA_IAM_INSTANCE_HOST }}
+ MB_MYSQL_AWS_IAM_TEST_PORT: 3306
+ MB_MYSQL_AWS_IAM_TEST_USER: my_iam_user
+ MB_MYSQL_AWS_IAM_TEST_DBNAME: test_db
+ MB_MYSQL_AWS_IAM_TEST_SSL_CERT: trust
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AURORA_IAM_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AURORA_IAM_SECRET_ACCESS_KEY }}
# for historic reasons (I don't want to mess around with required jobs) the job name should be something like
# "be-tests-mariadb-10-2-ee (0)"
name: "${{ matrix.version.name }} ${{ matrix.job.name }}"
@@ -786,11 +798,13 @@ jobs:
docker-image: postgres:12-alpine
env:
enable-ssl-tests: 'false'
+ enable-aws-iam-tests: 'false'
- name: Postgres Latest
junit-name: postgres-latest-ee
docker-image: postgres:latest
env:
enable-ssl-tests: 'true'
+ enable-aws-iam-tests: 'true'
job:
- name: Driver Tests
test-args: >-
@@ -830,6 +844,13 @@ jobs:
MB_POSTGRES_SSL_TEST_SSL: ${{ matrix.version.env.enable-ssl-tests }}
MB_POSTGRES_SSL_TEST_SSL_MODE: verify-full
MB_POSTGRES_SSL_TEST_SSL_ROOT_CERT_PATH: 'test-resources/certificates/us-east-2-bundle.pem'
+ MB_POSTGRES_AWS_IAM_TEST: ${{ matrix.version.env.enable-aws-iam-tests }}
+ MB_POSTGRES_AWS_IAM_TEST_HOST: ${{ secrets.POSTGRES_AURORA_IAM_INSTANCE_HOST }}
+ MB_POSTGRES_AWS_IAM_TEST_PORT: 5432
+ MB_POSTGRES_AWS_IAM_TEST_USER: my_iam_user
+ MB_POSTGRES_AWS_IAM_TEST_DBNAME: test_db
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AURORA_IAM_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AURORA_IAM_SECRET_ACCESS_KEY }}
services:
postgres:
image: ${{ matrix.version.docker-image }}
diff --git a/deps.edn b/deps.edn
index ecf7cf26375..4b0268a6be1 100644
--- a/deps.edn
+++ b/deps.edn
@@ -201,6 +201,7 @@
slingshot/slingshot {:mvn/version "0.12.2"} ; enhanced throw/catch, used by other deps
software.amazon.awssdk/s3 {:mvn/version "2.36.2"} ; AWS SDK v2 for S3
software.amazon.awssdk/sts {:mvn/version "2.36.2"} ; Required for S3 client to use service account
+ software.amazon.jdbc/aws-advanced-jdbc-wrapper {:mvn/version "2.6.5"} ; AWS JDBC wrapper for IAM authentication
stencil/stencil {:mvn/version "0.5.0"} ; Mustache templates for Clojure
user-agent/user-agent {:mvn/version "0.1.1"} ; User-Agent string parser, for Login History page & elsewhere
weavejester/dependency {:mvn/version "1.0.0"} ; Dependency graphs and topological sorting
diff --git a/src/metabase/app_db/data_source.clj b/src/metabase/app_db/data_source.clj
index 96ffd4f4d41..5e1a276efcc 100644
--- a/src/metabase/app_db/data_source.clj
+++ b/src/metabase/app_db/data_source.clj
@@ -100,9 +100,11 @@
(defn raw-connection-string->DataSource
"Return a [[javax.sql.DataSource]] given a raw JDBC connection string."
(^javax.sql.DataSource [s]
- (raw-connection-string->DataSource s nil nil nil))
+ (raw-connection-string->DataSource s nil nil nil nil))
(^javax.sql.DataSource [s username password azure-managed-identity-client-id]
+ (raw-connection-string->DataSource s username password azure-managed-identity-client-id nil))
+ (^javax.sql.DataSource [s username password azure-managed-identity-client-id aws-iam]
{:pre [(string? s)]}
;; normalize the protocol in case someone is trying to trip us up. Heroku is known for this and passes stuff in
;; like `postgres:...` to screw with us.
@@ -138,7 +140,11 @@
(seq password) (assoc :password password)
(and (empty? password)
(seq azure-managed-identity-client-id)) (assoc :azure-managed-identity-client-id
- azure-managed-identity-client-id))]
+ azure-managed-identity-client-id))
+ [s m] (if aws-iam
+ [(str/replace s #"^jdbc:(postgresql|mysql):" "jdbc:aws-wrapper:$1:")
+ (assoc m :wrapperPlugins "iam" :useSSL true)]
+ [s m])]
(update-h2/update-if-needed! s)
(->DataSource s (some-> (not-empty m) connection-pool/map->properties)))))
diff --git a/src/metabase/app_db/env.clj b/src/metabase/app_db/env.clj
index 7d0ee85d34f..45e2d5da1b7 100644
--- a/src/metabase/app_db/env.clj
+++ b/src/metabase/app_db/env.clj
@@ -77,7 +77,7 @@
(defn- broken-out-details
"Connection details that can be used when pretending the Metabase DB is itself a `Database` (e.g., to use the Generic
SQL driver functions on the Metabase DB itself)."
- [db-type {:keys [mb-db-dbname mb-db-host mb-db-pass mb-db-port mb-db-user mb-db-azure-managed-identity-client-id]
+ [db-type {:keys [mb-db-dbname mb-db-host mb-db-pass mb-db-port mb-db-user mb-db-azure-managed-identity-client-id mb-db-aws-iam mb-db-ssl-cert]
:as env-vars}]
(if (= db-type :h2)
(assoc h2-connection-properties
@@ -87,13 +87,15 @@
:db mb-db-dbname
:user mb-db-user
:password mb-db-pass
- :azure-managed-identity-client-id mb-db-azure-managed-identity-client-id}))
+ :azure-managed-identity-client-id mb-db-azure-managed-identity-client-id
+ :aws-iam mb-db-aws-iam
+ :ssl-cert mb-db-ssl-cert}))
(defn- env->DataSource
- [db-type {:keys [mb-db-connection-uri mb-db-user mb-db-pass mb-db-azure-managed-identity-client-id], :as env-vars}]
+ [db-type {:keys [mb-db-connection-uri mb-db-user mb-db-pass mb-db-azure-managed-identity-client-id aws-iam], :as env-vars}]
(if mb-db-connection-uri
(mdb.data-source/raw-connection-string->DataSource
- mb-db-connection-uri mb-db-user mb-db-pass mb-db-azure-managed-identity-client-id)
+ mb-db-connection-uri mb-db-user mb-db-pass mb-db-azure-managed-identity-client-id aws-iam)
(mdb.data-source/broken-out-details->DataSource db-type (broken-out-details db-type env-vars))))
;;;; exports: [[db-type]], [[db-file]], and [[data-source]] created using environment variables.
@@ -131,7 +133,9 @@
:mb-db-dbname (config/config-str :mb-db-dbname)
:mb-db-user (config/config-str :mb-db-user)
:mb-db-pass (config/config-str :mb-db-pass)
- :mb-db-azure-managed-identity-client-id (config/config-str :mb-db-azure-managed-identity-client-id)}
+ :mb-db-azure-managed-identity-client-id (config/config-str :mb-db-azure-managed-identity-client-id)
+ :mb-db-aws-iam (config/config-bool :mb-db-aws-iam)
+ :mb-db-ssl-cert (config/config-str :mb-db-ssl-cert)}
(env-defaults db-type)))
(def env
@@ -174,6 +178,26 @@
"If Metabase fails to launch, please add it and try again."
"See https://github.com/metabase/metabase/issues/8908 for more details."]))))
+;; If someone is using mysql and wants to authenticate with AWS IAM, they need to either specify the server ssl certificate
+;; or trust it. Let's let them know in case they don't.
+(when (and (:aws-iam env)
+ (= db-type :mysql))
+ (let [raw-connection-string (not-empty (:mb-db-connection-uri env))
+ ssl-cert (:mb-db-ssl-cert env)
+ has-ssl-config? (or ssl-cert
+ (and raw-connection-string
+ (or (str/includes? raw-connection-string "trustServerCertificate=true")
+ (and (str/includes? raw-connection-string "sslMode=VERIFY_CA")
+ (str/includes? raw-connection-string "serverSslCert=")))))]
+ (when-not has-ssl-config?
+ ;; Unfortunately this can't be i18n'ed because the application DB hasn't been initialized yet at the time we log
+ ;; this and thus the site locale is unavailable.
+ (log/warn (str/join " " ["Warning: MySQL with AWS IAM authentication detected but SSL configuration is missing."
+ (if raw-connection-string
+ "Add one of: `?sslMode=VERIFY_CA&serverSslCert=/path/to/certificate.pem` or `?trustServerCertificate=true` to your connection URI."
+ "Set MB_DB_SSL_CERT environment variable to either 'trust' or a path to your SSL certificate.")
+ "SSL certificates can be found at: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html"])))))
+
(def ^javax.sql.DataSource data-source
"A [[javax.sql.DataSource]] ultimately derived from the environment variables."
(env->DataSource db-type env))
diff --git a/src/metabase/app_db/spec.clj b/src/metabase/app_db/spec.clj
index a79629d9eaf..14ecbf6bd68 100644
--- a/src/metabase/app_db/spec.clj
+++ b/src/metabase/app_db/spec.clj
@@ -28,8 +28,14 @@
[host port db]
(str "//" (when-not (str/blank? host) (str host ":" port)) (if-not (str/blank? db) (str "/" db) "/")))
+(defn- make-aws-iam-spec [subprotocol]
+ {:subprotocol (str "aws-wrapper:" subprotocol)
+ :classname "software.amazon.jdbc.ds.AwsWrapperDataSource"
+ :useSSL true
+ :wrapperPlugins "iam"})
+
(defmethod spec :postgres
- [_ {:keys [host port db]
+ [_ {:keys [host port db aws-iam]
:or {host "localhost", port 5432, db ""}
:as opts}]
(merge
@@ -39,17 +45,32 @@
;; I think this is done to prevent conflicts with redshift driver registering itself to handle postgres://
:OpenSourceSubProtocolOverride true
:ApplicationName config/mb-version-and-process-identifier}
- (dissoc opts :host :port :db)))
+ (when aws-iam
+ (make-aws-iam-spec "postgresql"))
+ (dissoc opts :host :port :db :aws-iam)))
(defmethod spec :mysql
- [_ {:keys [host port db]
+ [_ {:keys [host port db aws-iam ssl-cert]
:or {host "localhost", port 3306, db ""}
:as opts}]
(merge
{:classname "org.mariadb.jdbc.Driver"
:subprotocol "mysql"
:subname (make-subname host (or port 3306) db)}
- (dissoc opts :host :port :db)))
+ (when aws-iam
+ (merge
+ (make-aws-iam-spec "mysql")
+ (cond
+ (= ssl-cert "trust")
+ {:trustServerCertificate true}
+
+ (and ssl-cert (not= ssl-cert "trust"))
+ {:sslMode "VERIFY_CA"
+ :serverSslCert ssl-cert}
+
+ :else
+ {:sslMode "VERIFY_CA"})))
+ (dissoc opts :host :port :db :aws-iam :ssl-cert)))
;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
;; !! !!
diff --git a/src/metabase/driver/common.clj b/src/metabase/driver/common.clj
index 759a97078c0..42aaafab956 100644
--- a/src/metabase/driver/common.clj
+++ b/src/metabase/driver/common.clj
@@ -249,36 +249,50 @@
added to the plugin manifest as connection properties, similar to the keys in the `default-options` map."
{:cloud-ip-address-info cloud-ip-address-info})
-(def auth-provider-options
- "Options for using an auth provider instead of a literal password."
- [{:name "use-auth-provider"
- :type :checked-section
- :check (fn []
- (and
- ;; Managed Identities only make sense if Metabase is in the same cloud as the DW
- (not (premium-features/is-hosted?))
- (premium-features/enable-database-auth-providers?)))
- :default false}
- {:name "auth-provider"
- :display-name (deferred-tru "Auth provider")
- :type :select
- :options [{:name (deferred-tru "Azure Managed Identity")
- :value "azure-managed-identity"}
- {:name (deferred-tru "OAuth")
- :value "oauth"}]
- :default "azure-managed-identity"
- :visible-if {"use-auth-provider" true}}
- {:name "azure-managed-identity-client-id"
- :display-name (deferred-tru "Client ID")
- :required true
- :visible-if {"auth-provider" "azure-managed-identity"}}
- {:name "oauth-token-url"
- :display-name (deferred-tru "Auth token URL")
- :required true
- :visible-if {"auth-provider" "oauth"}}
- {:name "oauth-token-headers"
- :display-name (deferred-tru "Auth token request headers (a JSON map)")
- :visible-if {"auth-provider" "oauth"}}])
+(defn auth-provider-options
+ "Options for using an auth provider instead of a literal password.
+ When called with no arguments, returns options for all available auth providers.
+ When called with a collection of provider keywords (e.g., #{:aws-iam}), returns options
+ filtered to only those providers."
+ ([]
+ (auth-provider-options nil))
+ ([allowed-providers]
+ (let [all-provider-options [{:name (deferred-tru "Azure Managed Identity")
+ :value "azure-managed-identity"}
+ {:name (deferred-tru "AWS IAM")
+ :value "aws-iam"}
+ {:name (deferred-tru "OAuth")
+ :value "oauth"}]
+ provider-options (if (seq allowed-providers)
+ (let [allowed-set (set (map name allowed-providers))]
+ (filterv #(contains? allowed-set (:value %)) all-provider-options))
+ all-provider-options)
+ default-provider (:value (first provider-options))]
+ [{:name "use-auth-provider"
+ :type :checked-section
+ :check (fn []
+ (and
+ ;; Managed Identities only make sense if Metabase is in the same cloud as the DW
+ (not (premium-features/is-hosted?))
+ (premium-features/enable-database-auth-providers?)))
+ :default false}
+ {:name "auth-provider"
+ :display-name (deferred-tru "Auth provider")
+ :type :select
+ :options provider-options
+ :default default-provider
+ :visible-if {"use-auth-provider" true}}
+ {:name "azure-managed-identity-client-id"
+ :display-name (deferred-tru "Client ID")
+ :required true
+ :visible-if {"auth-provider" "azure-managed-identity"}}
+ {:name "oauth-token-url"
+ :display-name (deferred-tru "Auth token URL")
+ :required true
+ :visible-if {"auth-provider" "oauth"}}
+ {:name "oauth-token-headers"
+ :display-name (deferred-tru "Auth token request headers (a JSON map)")
+ :visible-if {"auth-provider" "oauth"}}])))
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | Class -> Base Type |
diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.clj
index 834e7ef0a79..037be11fb06 100644
--- a/src/metabase/driver/mysql.clj
+++ b/src/metabase/driver/mysql.clj
@@ -227,7 +227,9 @@
(assoc driver.common/default-port-details :placeholder 3306)
driver.common/default-dbname-details
driver.common/default-user-details
- driver.common/default-password-details
+ (driver.common/auth-provider-options #{:aws-iam})
+ (assoc driver.common/default-password-details
+ :visible-if {"use-auth-provider" false})
driver.common/default-role-details
driver.common/cloud-ip-address-info
driver.common/default-ssl-details
@@ -651,23 +653,42 @@
(set-prog-nm-fn)))) ; additional-options did not contain connectionAttributes at all; set it
(defmethod sql-jdbc.conn/connection-details->spec :mysql
- [_ {ssl? :ssl, :keys [additional-options ssl-cert], :as details}]
+ [_ {ssl? :ssl, :keys [additional-options ssl-cert auth-provider], :as details}]
;; In versions older than 0.32.0 the MySQL driver did not correctly save `ssl?` connection status. Users worked
;; around this by including `useSSL=true`. Check if that's there, and if it is, assume SSL status. See #9629
;;
;; TODO - should this be fixed by a data migration instead?
(let [addl-opts-map (sql-jdbc.common/additional-options->map additional-options :url "=" false)
+ use-iam? (= (some-> auth-provider keyword) :aws-iam)
ssl? (or ssl? (= "true" (get addl-opts-map "useSSL")))
ssl-cert? (and ssl? (some? ssl-cert))]
(when (and ssl? (not (contains? addl-opts-map "trustServerCertificate")))
(log/info "You may need to add 'trustServerCertificate=true' to the additional connection options to connect with SSL."))
+ (when (and use-iam? (not ssl?))
+ (throw (ex-info "You must enable SSL in order to use AWS IAM authentication" {})))
+ (when (and use-iam?
+ (contains? addl-opts-map "sslMode")
+ (not (contains? addl-opts-map "sslMode=VERIFY_CA")))
+ (throw (ex-info "sslMode must be VERIFY_CA in order to use AWS IAM authentication" {})))
(merge
default-connection-args
;; newer versions of MySQL will complain if you don't specify this when not using SSL
{:useSSL (boolean ssl?)}
- (let [details (-> (if ssl-cert? (set/rename-keys details {:ssl-cert :serverSslCert}) details)
- (set/rename-keys {:dbname :db})
- (dissoc :ssl))]
+ (let [details (cond-> details
+ ssl-cert?
+ (set/rename-keys {:ssl-cert :serverSslCert})
+
+ use-iam?
+ (->
+ (assoc :subprotocol "aws-wrapper:mysql"
+ :classname "software.amazon.jdbc.ds.AwsWrapperDataSource"
+ :sslMode "VERIFY_CA"
+ :wrapperPlugins "iam")
+ (dissoc :auth-provider :use-auth-provider))
+
+ true
+ (-> (set/rename-keys {:dbname :db})
+ (dissoc :ssl)))]
(-> (driver-api/spec :mysql details)
(maybe-add-program-name-option addl-opts-map)
(sql-jdbc.common/handle-additional-options details))))))
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index 947c4846043..86e738364c8 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -158,7 +158,7 @@
(assoc driver.common/default-port-details :placeholder 5432)
driver.common/default-dbname-details
driver.common/default-user-details
- driver.common/auth-provider-options
+ (driver.common/auth-provider-options)
(assoc driver.common/default-password-details
:visible-if {"use-auth-provider" false})
driver.common/cloud-ip-address-info
@@ -920,8 +920,9 @@
{:sslmode "disable"})
(defmethod sql-jdbc.conn/connection-details->spec :postgres
- [_ {ssl? :ssl, :as details-map}]
- (let [props (-> details-map
+ [_ {ssl? :ssl, :keys [auth-provider], :as details-map}]
+ (let [use-iam? (= (some-> auth-provider keyword) :aws-iam)
+ props (-> details-map
(update :port (fn [port]
(if (string? port)
(Integer/parseInt port)
@@ -937,10 +938,19 @@
;; internal property values back; only merge in the ones the driver might recognize
(merge ssl-prms (select-keys props (keys ssl-prms))))
(merge disable-ssl-params props))
+ props (if use-iam?
+ (-> props
+ (assoc :subprotocol "aws-wrapper:postgresql"
+ :classname "software.amazon.jdbc.ds.AwsWrapperDataSource"
+ :wrapperPlugins "iam")
+ (dissoc :auth-provider :use-auth-provider))
+ props)
props (as-> props it
(set/rename-keys it {:dbname :db})
(driver-api/spec :postgres it)
(sql-jdbc.common/handle-additional-options it details-map))]
+ (when (and use-iam? (not ssl?))
+ (throw (ex-info "You must enable SSL in order to use AWS IAM authentication" {})))
props))
(defmethod sql-jdbc.sync/excluded-schemas :postgres [_driver] #{"information_schema" "pg_catalog"})
diff --git a/src/metabase/driver/sql_jdbc/common.clj b/src/metabase/driver/sql_jdbc/common.clj
index 2c2bcede538..b543079dbe3 100644
--- a/src/metabase/driver/sql_jdbc/common.clj
+++ b/src/metabase/driver/sql_jdbc/common.clj
@@ -43,7 +43,7 @@
v)) "" additional-opts)))
(defn handle-additional-options
- "If `details` contains an `:addtional-options` key, append those options to the connection string in `connection-spec`.
+ "If `details` contains an `:additional-options` key, append those options to the connection string in `connection-spec`.
(Some drivers like MySQL provide this details field to allow special behavior where needed).
Optionally specify `seperator-style`, which defaults to `:url` (e.g. `?a=1&b=2`). You may instead set it to
diff --git a/test/metabase/app_db/aws_iam_test.clj b/test/metabase/app_db/aws_iam_test.clj
new file mode 100644
index 00000000000..ac1ebf81f2d
--- /dev/null
+++ b/test/metabase/app_db/aws_iam_test.clj
@@ -0,0 +1,114 @@
+(ns metabase.app-db.aws-iam-test
+ "Integration tests for AWS IAM authentication with application database.
+
+ These tests require actual AWS RDS instances configured for IAM authentication.
+ Tests will be skipped if required environment variables are not set.
+
+ Required environment variables:
+ - Postgres:
+ - MB_POSTGRES_AWS_IAM_TEST_HOST
+ - MB_POSTGRES_AWS_IAM_TEST_PORT
+ - MB_POSTGRES_AWS_IAM_TEST_USER
+ - MB_POSTGRES_AWS_IAM_TEST_DBNAME
+ - MySQL:
+ - MB_MYSQL_AWS_IAM_TEST_HOST
+ - MB_MYSQL_AWS_IAM_TEST_PORT
+ - MB_MYSQL_AWS_IAM_TEST_USER
+ - MB_MYSQL_AWS_IAM_TEST_DBNAME
+ - MB_MYSQL_AWS_IAM_TEST_SSL_CERT (required, PEM certificate content/trust)"
+ (:require
+ [clojure.test :refer :all]
+ [metabase.app-db.data-source :as mdb.data-source]
+ [metabase.config.core :as config]
+ [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(defn- get-connection-and-close!
+ "Attempt to get a connection from the DataSource and close it immediately.
+ Returns true if successful, throws if connection fails."
+ [^javax.sql.DataSource datasource]
+ (with-open [conn (.getConnection datasource)]
+ (assert conn "Connection should not be nil")
+ true))
+
+(deftest ^:mb/once postgres-aws-iam-test
+ (if (config/config-bool :mb-postgres-aws-iam-test)
+ (let [host (config/config-str :mb-postgres-aws-iam-test-host)
+ port (config/config-int :mb-postgres-aws-iam-test-port)
+ user (config/config-str :mb-postgres-aws-iam-test-user)
+ dbname (config/config-str :mb-postgres-aws-iam-test-dbname)
+ uri (format "postgres://%s:%d/%s?user=%s" host port dbname user)]
+ (testing "Connection details are configured"
+ (is (string? host))
+ (is (string? user))
+ (is (int? port))
+ (is (string? dbname)))
+
+ (testing "using broken-out details"
+ (testing "Can create DataSource with AWS IAM enabled"
+ (let [details {:host host
+ :port port
+ :user user
+ :db dbname
+ :aws-iam true}
+ datasource (mdb.data-source/broken-out-details->DataSource :postgres details)]
+ (is (instance? javax.sql.DataSource datasource))
+
+ (testing "Can establish connection using AWS IAM"
+ (is (true? (get-connection-and-close! datasource)))))))
+
+ (testing "using uri"
+ (testing "Can create DataSource with AWS IAM enabled"
+ (let [datasource (mdb.data-source/raw-connection-string->DataSource
+ uri nil nil nil true)]
+ (is (instance? javax.sql.DataSource datasource))
+
+ (testing "Can establish connection using AWS IAM"
+ (is (true? (get-connection-and-close! datasource))))))))
+ (log/info "Skipping test: MB_POSTGRES_AWS_IAM_TEST not set")))
+
+(deftest ^:mb/once mysql-aws-iam-test
+ (testing "MySQL App DB connection with AWS IAM authentication"
+ (if (config/config-bool :mb-mysql-aws-iam-test)
+ (let [host (config/config-str :mb-mysql-aws-iam-test-host)
+ port (config/config-int :mb-mysql-aws-iam-test-port)
+ user (config/config-str :mb-mysql-aws-iam-test-user)
+ dbname (config/config-str :mb-mysql-aws-iam-test-dbname)
+ ssl-cert (config/config-str :mb-mysql-aws-iam-test-ssl-cert)
+ uri (format "mysql://%s:%d/%s?user=%s&%s" host port dbname user
+ (if (= ssl-cert "trust")
+ "trustServerCertificate=true"
+ (str "?sslMode=VERIFY_CA&serverSslCert=" ssl-cert)))]
+ (testing "Connection details are configured"
+ (is (string? host))
+ (is (string? user))
+ (is (int? port))
+ (is (string? dbname)))
+
+ (testing "SSL certificate is configured"
+ (is (string? ssl-cert)))
+
+ (testing "using broken-out details"
+ (testing "Can create DataSource with AWS IAM enabled"
+ (let [details {:host host
+ :port port
+ :user user
+ :db dbname
+ :ssl-cert ssl-cert
+ :aws-iam true}
+ datasource (mdb.data-source/broken-out-details->DataSource :mysql details)]
+ (is (instance? javax.sql.DataSource datasource))
+
+ (testing "Can establish connection using AWS IAM"
+ (is (true? (get-connection-and-close! datasource)))))))
+
+ (testing "using uri"
+ (testing "Can create DataSource with AWS IAM enabled"
+ (let [datasource (mdb.data-source/raw-connection-string->DataSource
+ uri nil nil nil true)]
+ (is (instance? javax.sql.DataSource datasource))
+
+ (testing "Can establish connection using AWS IAM"
+ (is (true? (get-connection-and-close! datasource))))))))
+ (log/info "Skipping test: MB_MYSQL_AWS_IAM_TEST not set"))))
diff --git a/test/metabase/app_db/data_source_test.clj b/test/metabase/app_db/data_source_test.clj
index ada307cb8e4..89ebfc3f47a 100644
--- a/test/metabase/app_db/data_source_test.clj
+++ b/test/metabase/app_db/data_source_test.clj
@@ -84,6 +84,70 @@
(is (= [{:one 1}]
(jdbc/query {:connection conn} "SELECT 1 AS one;")))))))
+(deftest ^:parallel broken-out-details-test-6
+ (testing :aws-iam
+ (testing "Postgres with AWS IAM"
+ (is (= (->DataSource
+ "jdbc:aws-wrapper:postgresql://localhost:5432/metabase"
+ {"ApplicationName" config/mb-version-and-process-identifier
+ "OpenSourceSubProtocolOverride" "true"
+ "user" "cam"
+ "useSSL" true
+ "wrapperPlugins" "iam"})
+ (mdb.data-source/broken-out-details->DataSource :postgres {:host "localhost"
+ :port 5432
+ :user "cam"
+ :aws-iam true
+ :db "metabase"}))))))
+
+(deftest ^:parallel broken-out-details-test-7
+ (testing :aws-iam
+ (testing "MySQL with AWS IAM"
+ (is (= (->DataSource
+ "jdbc:aws-wrapper:mysql://localhost:3306/metabase"
+ {"user" "root"
+ "wrapperPlugins" "iam"
+ "useSSL" true
+ "sslMode" "VERIFY_CA"})
+ (mdb.data-source/broken-out-details->DataSource :mysql {:host "localhost"
+ :port 3306
+ :user "root"
+ :aws-iam true
+ :db "metabase"}))))))
+
+(deftest ^:parallel broken-out-details-test-8
+ (testing :aws-iam
+ (testing "MySQL with AWS IAM and ssl-cert=trust"
+ (is (= (->DataSource
+ "jdbc:aws-wrapper:mysql://localhost:3306/metabase"
+ {"user" "root"
+ "wrapperPlugins" "iam"
+ "useSSL" true
+ "trustServerCertificate" "true"})
+ (mdb.data-source/broken-out-details->DataSource :mysql {:host "localhost"
+ :port 3306
+ :user "root"
+ :aws-iam true
+ :ssl-cert "trust"
+ :db "metabase"}))))))
+
+(deftest ^:parallel broken-out-details-test-9
+ (testing :aws-iam
+ (testing "MySQL with AWS IAM and ssl-cert path"
+ (is (= (->DataSource
+ "jdbc:aws-wrapper:mysql://localhost:3306/metabase"
+ {"user" "root"
+ "wrapperPlugins" "iam"
+ "sslMode" "VERIFY_CA"
+ "useSSL" true
+ "serverSslCert" "/path/to/certificate.pem"})
+ (mdb.data-source/broken-out-details->DataSource :mysql {:host "localhost"
+ :port 3306
+ :user "root"
+ :aws-iam true
+ :ssl-cert "/path/to/certificate.pem"
+ :db "metabase"}))))))
+
(deftest ^:parallel connection-string-test
(let [data-source (mdb.data-source/raw-connection-string->DataSource
(format "jdbc:h2:mem:%s" (mt/random-name)))]
@@ -169,6 +233,24 @@
(mdb.data-source/raw-connection-string->DataSource "postgres://metabase" "" "1234" nil)
(mdb.data-source/raw-connection-string->DataSource "postgres://metabase" "" "1234" "client ID"))))))
+(deftest ^:parallel raw-connection-string-with-aws-iam-test
+ (testing "Raw connection string with AWS IAM enabled for Postgres"
+ (is (= (->DataSource
+ "jdbc:aws-wrapper:postgresql://metabase"
+ {"user" "cam"
+ "useSSL" true
+ "wrapperPlugins" "iam"})
+ (mdb.data-source/raw-connection-string->DataSource "postgres://metabase" "cam" nil nil true)))))
+
+(deftest ^:parallel raw-connection-string-with-aws-iam-test-2
+ (testing "Raw connection string with AWS IAM enabled for MySQL"
+ (is (= (->DataSource
+ "jdbc:aws-wrapper:mysql://metabase"
+ {"user" "cam"
+ "useSSL" true
+ "wrapperPlugins" "iam"})
+ (mdb.data-source/raw-connection-string->DataSource "mysql://metabase" "cam" nil nil true)))))
+
(deftest ^:parallel equality-test
(testing "Two DataSources with the same URL should be equal"
(is (= (mdb.data-source/raw-connection-string->DataSource "ABCD")
diff --git a/test/metabase/driver/sql_jdbc/connection_test.clj b/test/metabase/driver/sql_jdbc/connection_test.clj
index 3489cec120b..0caea3630b0 100644
--- a/test/metabase/driver/sql_jdbc/connection_test.clj
+++ b/test/metabase/driver/sql_jdbc/connection_test.clj
@@ -18,6 +18,7 @@
[metabase.driver.sql-jdbc.connection.ssh-tunnel-test :as ssh-test]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.util :as driver.u]
+ [metabase.premium-features.core :as premium-features]
[metabase.query-processor :as qp]
[metabase.query-processor.test-util :as qp.test-util]
[metabase.sync.core :as sync]
@@ -28,6 +29,7 @@
[metabase.test.util :as tu]
[metabase.util :as u]
[metabase.util.http :as u.http]
+ [metabase.util.log :as log]
[next.jdbc :as next.jdbc]
[toucan2.core :as t2])
(:import
@@ -358,6 +360,63 @@
;; we must have created more than one connection
(is (> @connection-creations 1))))))))))
+#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}
+(deftest test-aws-iam-auth-provider-connection
+ (mt/with-premium-features #{:database-auth-providers}
+ (testing "AWS IAM authentication for Postgres"
+ (mt/test-driver :postgres
+ (let [db-details (:details (mt/db))
+ iam-db-details (-> db-details
+ (dissoc :password)
+ (assoc :use-auth-provider true
+ :auth-provider :aws-iam
+ :ssl true))]
+ (testing "Connection spec is configured with AWS wrapper"
+ (let [spec (sql-jdbc.conn/connection-details->spec :postgres iam-db-details)]
+ (is (= "aws-wrapper:postgresql" (:subprotocol spec)))
+ (is (= "software.amazon.jdbc.ds.AwsWrapperDataSource" (:classname spec)))
+ (is (= "iam" (:wrapperPlugins spec))))))))
+ (testing "AWS IAM authentication for MySQL"
+ (mt/test-driver :mysql
+ (let [db-details (:details (mt/db))
+ iam-db-details (-> db-details
+ (dissoc :password)
+ (assoc :use-auth-provider true
+ :auth-provider :aws-iam
+ :ssl true))]
+ (testing "Connection spec is configured with AWS wrapper"
+ (let [spec (sql-jdbc.conn/connection-details->spec :mysql iam-db-details)]
+ (is (= "aws-wrapper:mysql" (:subprotocol spec)))
+ (is (= "software.amazon.jdbc.ds.AwsWrapperDataSource" (:classname spec)))
+ (is (= "iam" (:wrapperPlugins spec)))
+ (is (= "VERIFY_CA" (:sslMode spec))))))))))
+
+#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}
+(deftest ^:parallel test-aws-iam-requires-ssl
+ (testing "AWS IAM authentication requires SSL to be enabled"
+ (testing "Postgres throws error when SSL is disabled"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"You must enable SSL in order to use AWS IAM authentication"
+ (sql-jdbc.conn/connection-details->spec :postgres
+ {:host "localhost"
+ :port 5432
+ :user "cam"
+ :auth-provider :aws-iam
+ :ssl false
+ :db "metabase"}))))
+ (testing "MySQL throws error when SSL is disabled"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"You must enable SSL in order to use AWS IAM authentication"
+ (sql-jdbc.conn/connection-details->spec :mysql
+ {:host "localhost"
+ :port 3306
+ :user "root"
+ :auth-provider :aws-iam
+ :ssl false
+ :db "metabase"}))))))
+
(defmacro ^:private with-tunnel-details!
[& body]
`(let [original-details# (:details (mt/db))
@@ -508,3 +567,58 @@
;; check the query again; the tunnel should have been reestablished
(check-data))))
(finally (.stop ^Server server))))))))
+
+#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}
+(deftest postgres-aws-iam-can-connect
+ (if (config/config-bool :mb-postgres-aws-iam-test)
+ (let [host (config/config-str :mb-postgres-aws-iam-test-host)
+ port (config/config-int :mb-postgres-aws-iam-test-port)
+ user (config/config-str :mb-postgres-aws-iam-test-user)
+ dbname (config/config-str :mb-postgres-aws-iam-test-dbname)]
+ (with-redefs [premium-features/is-hosted? (constantly false)]
+ (testing "Connection details are configured"
+ (is (string? host))
+ (is (string? user))
+ (is (int? port))
+ (is (string? dbname)))
+
+ (mt/with-temporary-setting-values [db-connection-timeout-ms 10000]
+ (is
+ (driver.u/can-connect-with-details? :postgres {:host host
+ :port port
+ :dbname dbname
+ :user user
+ :use-auth-provider true
+ :auth-provider :aws-iam
+ :ssl true})))))
+ (log/info "Skipping test: MB_POSTGRES_AWS_IAM_TEST not set")))
+
+#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}
+(deftest mysql-aws-iam-can-connect
+ (if (config/config-bool :mb-mysql-aws-iam-test)
+ (let [host (config/config-str :mb-mysql-aws-iam-test-host)
+ port (config/config-int :mb-mysql-aws-iam-test-port)
+ user (config/config-str :mb-mysql-aws-iam-test-user)
+ dbname (config/config-str :mb-mysql-aws-iam-test-dbname)
+ ssl-cert (config/config-str :mb-mysql-aws-iam-test-ssl-cert)]
+ (with-redefs [premium-features/is-hosted? (constantly false)]
+ (testing "Connection details are configured"
+ (is (string? host))
+ (is (string? user))
+ (is (int? port))
+ (is (string? dbname))
+ (is (string? ssl-cert)))
+
+ (mt/with-temporary-setting-values [db-connection-timeout-ms 10000]
+ (is
+ (driver.u/can-connect-with-details? :mysql {:host host
+ :port port
+ :dbname dbname
+ :user user
+ :additional-options (if (= ssl-cert "trust")
+ "trustServerCertificate=true"
+ (str "serverSslCert=" ssl-cert))
+ :use-auth-provider true
+ :auth-provider :aws-iam
+ :ssl true})))))
+ (log/info "Skipping test: MB_MYSQL_AWS_IAM_TEST not set")))