The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add AWS IAM Authentication Support for PostgreSQL and MySQL (#65418)
Nicola Momettoff099455d047
diff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.ymlindex dfda83f3fff..c88368d2a76 100644--- a/.github/workflows/app-db.yml+++ b/.github/workflows/app-db.yml@@ -137,11 +137,13 @@ jobs:image: cimg/mysql:8.0env:enable-ssl-tests: 'false'+ enable-aws-iam-tests: 'false'- name: MySQL Latestjunit-name: be-tests-mysql-latest-eeimage: mysql:latestenv:enable-ssl-tests: 'true'+ enable-aws-iam-tests: 'true'job:- name: Enterprise Testsbuild-static-viz: false@@ -211,6 +213,14 @@ jobs:MB_MYSQL_SSL_TEST_SSL_CERT: ${{ secrets.MB_MYSQL_SSL_TEST_SSL_CERT }}MB_MYSQL_SSL_TEST_USER: metabaseMB_MYSQL_SSL_TEST_PASSWORD: ${{ secrets.MYSQL_RDS_SSL_INSTANCE_PASSWORD }}+ MB_MYSQL_AWS_IAM_TEST: ${{ matrix.version.env.enable-aws-iam-tests }}+ MB_MYSQL_AWS_IAM_TEST_HOST: ${{ secrets.MYSQL_AURORA_IAM_INSTANCE_HOST }}+ MB_MYSQL_AWS_IAM_TEST_PORT: 3306+ MB_MYSQL_AWS_IAM_TEST_USER: my_iam_user+ MB_MYSQL_AWS_IAM_TEST_DBNAME: test_db+ MB_MYSQL_AWS_IAM_TEST_SSL_CERT: trust+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AURORA_IAM_ACCESS_KEY_ID }}+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AURORA_IAM_SECRET_ACCESS_KEY }}# for historic reasons (I don't want to mess around with required jobs) the job name should be something like# "be-tests-mariadb-10-2-ee (0)"name: "${{ matrix.version.name }} ${{ matrix.job.name }}"@@ -249,11 +259,13 @@ jobs:docker-image: postgres:12-alpineenv:enable-ssl-tests: 'false'+ enable-aws-iam-tests: 'false'- name: Postgres Latestjunit-name: postgres-latest-eedocker-image: postgres:latestenv:enable-ssl-tests: 'true'+ enable-aws-iam-tests: 'true'job:- name: Enterprise Testsbuild-static-viz: false@@ -305,6 +317,13 @@ jobs:MB_POSTGRES_SSL_TEST_SSL: ${{ matrix.version.env.enable-ssl-tests }}MB_POSTGRES_SSL_TEST_SSL_MODE: verify-fullMB_POSTGRES_SSL_TEST_SSL_ROOT_CERT_PATH: 'test-resources/certificates/us-east-2-bundle.pem'+ MB_POSTGRES_AWS_IAM_TEST: ${{ matrix.version.env.enable-aws-iam-tests }}+ MB_POSTGRES_AWS_IAM_TEST_HOST: ${{ secrets.POSTGRES_AURORA_IAM_INSTANCE_HOST }}+ MB_POSTGRES_AWS_IAM_TEST_PORT: 5432+ MB_POSTGRES_AWS_IAM_TEST_USER: my_iam_user+ MB_POSTGRES_AWS_IAM_TEST_DBNAME: test_db+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AURORA_IAM_ACCESS_KEY_ID }}+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AURORA_IAM_SECRET_ACCESS_KEY }}services:postgres:image: ${{ matrix.version.docker-image }}diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 9acb6a88039..f4d29700768 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -423,21 +423,25 @@ jobs:image: circleci/mariadb:10.2.23env:enable-ssl-tests: 'false'+ enable-aws-iam-tests: 'false'- name: MariaDB Latestjunit-name: be-tests-mariadb-latest-eeimage: circleci/mariadb:latestenv:enable-ssl-tests: 'false'+ enable-aws-iam-tests: 'false'- name: MySQL 8.0junit-name: be-tests-mysql-8-0-eeimage: mysql:8.0env:enable-ssl-tests: 'false'+ enable-aws-iam-tests: 'false'- name: MySQL Latestjunit-name: be-tests-mysql-latest-eeimage: mysql:latestenv:enable-ssl-tests: 'true'+ enable-aws-iam-tests: 'true'job:- name: Driver Teststest-args: >-@@ -496,6 +500,14 @@ jobs:MB_MYSQL_SSL_TEST_SSL_CERT: ${{ secrets.MB_MYSQL_SSL_TEST_SSL_CERT }}MB_MYSQL_SSL_TEST_USER: metabaseMB_MYSQL_SSL_TEST_PASSWORD: ${{ secrets.MYSQL_RDS_SSL_INSTANCE_PASSWORD }}+ MB_MYSQL_AWS_IAM_TEST: ${{ matrix.version.env.enable-aws-iam-tests }}+ MB_MYSQL_AWS_IAM_TEST_HOST: ${{ secrets.MYSQL_AURORA_IAM_INSTANCE_HOST }}+ MB_MYSQL_AWS_IAM_TEST_PORT: 3306+ MB_MYSQL_AWS_IAM_TEST_USER: my_iam_user+ MB_MYSQL_AWS_IAM_TEST_DBNAME: test_db+ MB_MYSQL_AWS_IAM_TEST_SSL_CERT: trust+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AURORA_IAM_ACCESS_KEY_ID }}+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AURORA_IAM_SECRET_ACCESS_KEY }}# for historic reasons (I don't want to mess around with required jobs) the job name should be something like# "be-tests-mariadb-10-2-ee (0)"name: "${{ matrix.version.name }} ${{ matrix.job.name }}"@@ -786,11 +798,13 @@ jobs:docker-image: postgres:12-alpineenv:enable-ssl-tests: 'false'+ enable-aws-iam-tests: 'false'- name: Postgres Latestjunit-name: postgres-latest-eedocker-image: postgres:latestenv:enable-ssl-tests: 'true'+ enable-aws-iam-tests: 'true'job:- name: Driver Teststest-args: >-@@ -830,6 +844,13 @@ jobs:MB_POSTGRES_SSL_TEST_SSL: ${{ matrix.version.env.enable-ssl-tests }}MB_POSTGRES_SSL_TEST_SSL_MODE: verify-fullMB_POSTGRES_SSL_TEST_SSL_ROOT_CERT_PATH: 'test-resources/certificates/us-east-2-bundle.pem'+ MB_POSTGRES_AWS_IAM_TEST: ${{ matrix.version.env.enable-aws-iam-tests }}+ MB_POSTGRES_AWS_IAM_TEST_HOST: ${{ secrets.POSTGRES_AURORA_IAM_INSTANCE_HOST }}+ MB_POSTGRES_AWS_IAM_TEST_PORT: 5432+ MB_POSTGRES_AWS_IAM_TEST_USER: my_iam_user+ MB_POSTGRES_AWS_IAM_TEST_DBNAME: test_db+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_AURORA_IAM_ACCESS_KEY_ID }}+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_AURORA_IAM_SECRET_ACCESS_KEY }}services:postgres:image: ${{ matrix.version.docker-image }}diff --git a/deps.edn b/deps.ednindex ecf7cf26375..4b0268a6be1 100644--- a/deps.edn+++ b/deps.edn@@ -201,6 +201,7 @@slingshot/slingshot {:mvn/version "0.12.2"} ; enhanced throw/catch, used by other depssoftware.amazon.awssdk/s3 {:mvn/version "2.36.2"} ; AWS SDK v2 for S3software.amazon.awssdk/sts {:mvn/version "2.36.2"} ; Required for S3 client to use service account+ software.amazon.jdbc/aws-advanced-jdbc-wrapper {:mvn/version "2.6.5"} ; AWS JDBC wrapper for IAM authenticationstencil/stencil {:mvn/version "0.5.0"} ; Mustache templates for Clojureuser-agent/user-agent {:mvn/version "0.1.1"} ; User-Agent string parser, for Login History page & elsewhereweavejester/dependency {:mvn/version "1.0.0"} ; Dependency graphs and topological sortingdiff --git a/src/metabase/app_db/data_source.clj b/src/metabase/app_db/data_source.cljindex 96ffd4f4d41..5e1a276efcc 100644--- a/src/metabase/app_db/data_source.clj+++ b/src/metabase/app_db/data_source.clj@@ -100,9 +100,11 @@(defn raw-connection-string->DataSource"Return a [[javax.sql.DataSource]] given a raw JDBC connection string."(^javax.sql.DataSource [s]- (raw-connection-string->DataSource s nil nil nil))+ (raw-connection-string->DataSource s nil nil nil nil))(^javax.sql.DataSource [s username password azure-managed-identity-client-id]+ (raw-connection-string->DataSource s username password azure-managed-identity-client-id nil))+ (^javax.sql.DataSource [s username password azure-managed-identity-client-id aws-iam]{:pre [(string? s)]};; normalize the protocol in case someone is trying to trip us up. Heroku is known for this and passes stuff in;; like `postgres:...` to screw with us.@@ -138,7 +140,11 @@(seq password) (assoc :password password)(and (empty? password)(seq azure-managed-identity-client-id)) (assoc :azure-managed-identity-client-id- azure-managed-identity-client-id))]+ azure-managed-identity-client-id))+ [s m] (if aws-iam+ [(str/replace s #"^jdbc:(postgresql|mysql):" "jdbc:aws-wrapper:$1:")+ (assoc m :wrapperPlugins "iam" :useSSL true)]+ [s m])](update-h2/update-if-needed! s)(->DataSource s (some-> (not-empty m) connection-pool/map->properties)))))diff --git a/src/metabase/app_db/env.clj b/src/metabase/app_db/env.cljindex 7d0ee85d34f..45e2d5da1b7 100644--- a/src/metabase/app_db/env.clj+++ b/src/metabase/app_db/env.clj@@ -77,7 +77,7 @@(defn- broken-out-details"Connection details that can be used when pretending the Metabase DB is itself a `Database` (e.g., to use the GenericSQL driver functions on the Metabase DB itself)."- [db-type {:keys [mb-db-dbname mb-db-host mb-db-pass mb-db-port mb-db-user mb-db-azure-managed-identity-client-id]+ [db-type {:keys [mb-db-dbname mb-db-host mb-db-pass mb-db-port mb-db-user mb-db-azure-managed-identity-client-id mb-db-aws-iam mb-db-ssl-cert]:as env-vars}](if (= db-type :h2)(assoc h2-connection-properties@@ -87,13 +87,15 @@:db mb-db-dbname:user mb-db-user:password mb-db-pass- :azure-managed-identity-client-id mb-db-azure-managed-identity-client-id}))+ :azure-managed-identity-client-id mb-db-azure-managed-identity-client-id+ :aws-iam mb-db-aws-iam+ :ssl-cert mb-db-ssl-cert}))(defn- env->DataSource- [db-type {:keys [mb-db-connection-uri mb-db-user mb-db-pass mb-db-azure-managed-identity-client-id], :as env-vars}]+ [db-type {:keys [mb-db-connection-uri mb-db-user mb-db-pass mb-db-azure-managed-identity-client-id aws-iam], :as env-vars}](if mb-db-connection-uri(mdb.data-source/raw-connection-string->DataSource- mb-db-connection-uri mb-db-user mb-db-pass mb-db-azure-managed-identity-client-id)+ mb-db-connection-uri mb-db-user mb-db-pass mb-db-azure-managed-identity-client-id aws-iam)(mdb.data-source/broken-out-details->DataSource db-type (broken-out-details db-type env-vars))));;;; exports: [[db-type]], [[db-file]], and [[data-source]] created using environment variables.@@ -131,7 +133,9 @@:mb-db-dbname (config/config-str :mb-db-dbname):mb-db-user (config/config-str :mb-db-user):mb-db-pass (config/config-str :mb-db-pass)- :mb-db-azure-managed-identity-client-id (config/config-str :mb-db-azure-managed-identity-client-id)}+ :mb-db-azure-managed-identity-client-id (config/config-str :mb-db-azure-managed-identity-client-id)+ :mb-db-aws-iam (config/config-bool :mb-db-aws-iam)+ :mb-db-ssl-cert (config/config-str :mb-db-ssl-cert)}(env-defaults db-type)))(def env@@ -174,6 +178,26 @@"If Metabase fails to launch, please add it and try again.""See https://github.com/metabase/metabase/issues/8908 for more details."]))))+;; If someone is using mysql and wants to authenticate with AWS IAM, they need to either specify the server ssl certificate+;; or trust it. Let's let them know in case they don't.+(when (and (:aws-iam env)+ (= db-type :mysql))+ (let [raw-connection-string (not-empty (:mb-db-connection-uri env))+ ssl-cert (:mb-db-ssl-cert env)+ has-ssl-config? (or ssl-cert+ (and raw-connection-string+ (or (str/includes? raw-connection-string "trustServerCertificate=true")+ (and (str/includes? raw-connection-string "sslMode=VERIFY_CA")+ (str/includes? raw-connection-string "serverSslCert=")))))]+ (when-not has-ssl-config?+ ;; Unfortunately this can't be i18n'ed because the application DB hasn't been initialized yet at the time we log+ ;; this and thus the site locale is unavailable.+ (log/warn (str/join " " ["Warning: MySQL with AWS IAM authentication detected but SSL configuration is missing."+ (if raw-connection-string+ "Add one of: `?sslMode=VERIFY_CA&serverSslCert=/path/to/certificate.pem` or `?trustServerCertificate=true` to your connection URI."+ "Set MB_DB_SSL_CERT environment variable to either 'trust' or a path to your SSL certificate.")+ "SSL certificates can be found at: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html"])))))+(def ^javax.sql.DataSource data-source"A [[javax.sql.DataSource]] ultimately derived from the environment variables."(env->DataSource db-type env))diff --git a/src/metabase/app_db/spec.clj b/src/metabase/app_db/spec.cljindex a79629d9eaf..14ecbf6bd68 100644--- a/src/metabase/app_db/spec.clj+++ b/src/metabase/app_db/spec.clj@@ -28,8 +28,14 @@[host port db](str "//" (when-not (str/blank? host) (str host ":" port)) (if-not (str/blank? db) (str "/" db) "/")))+(defn- make-aws-iam-spec [subprotocol]+ {:subprotocol (str "aws-wrapper:" subprotocol)+ :classname "software.amazon.jdbc.ds.AwsWrapperDataSource"+ :useSSL true+ :wrapperPlugins "iam"})+(defmethod spec :postgres- [_ {:keys [host port db]+ [_ {:keys [host port db aws-iam]:or {host "localhost", port 5432, db ""}:as opts}](merge@@ -39,17 +45,32 @@;; I think this is done to prevent conflicts with redshift driver registering itself to handle postgres://:OpenSourceSubProtocolOverride true:ApplicationName config/mb-version-and-process-identifier}- (dissoc opts :host :port :db)))+ (when aws-iam+ (make-aws-iam-spec "postgresql"))+ (dissoc opts :host :port :db :aws-iam)))(defmethod spec :mysql- [_ {:keys [host port db]+ [_ {:keys [host port db aws-iam ssl-cert]:or {host "localhost", port 3306, db ""}:as opts}](merge{:classname "org.mariadb.jdbc.Driver":subprotocol "mysql":subname (make-subname host (or port 3306) db)}- (dissoc opts :host :port :db)))+ (when aws-iam+ (merge+ (make-aws-iam-spec "mysql")+ (cond+ (= ssl-cert "trust")+ {:trustServerCertificate true}++ (and ssl-cert (not= ssl-cert "trust"))+ {:sslMode "VERIFY_CA"+ :serverSslCert ssl-cert}++ :else+ {:sslMode "VERIFY_CA"})))+ (dissoc opts :host :port :db :aws-iam :ssl-cert)));; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!;; !! !!diff --git a/src/metabase/driver/common.clj b/src/metabase/driver/common.cljindex 759a97078c0..42aaafab956 100644--- a/src/metabase/driver/common.clj+++ b/src/metabase/driver/common.clj@@ -249,36 +249,50 @@added to the plugin manifest as connection properties, similar to the keys in the `default-options` map."{:cloud-ip-address-info cloud-ip-address-info})-(def auth-provider-options- "Options for using an auth provider instead of a literal password."- [{:name "use-auth-provider"- :type :checked-section- :check (fn []- (and- ;; Managed Identities only make sense if Metabase is in the same cloud as the DW- (not (premium-features/is-hosted?))- (premium-features/enable-database-auth-providers?)))- :default false}- {:name "auth-provider"- :display-name (deferred-tru "Auth provider")- :type :select- :options [{:name (deferred-tru "Azure Managed Identity")- :value "azure-managed-identity"}- {:name (deferred-tru "OAuth")- :value "oauth"}]- :default "azure-managed-identity"- :visible-if {"use-auth-provider" true}}- {:name "azure-managed-identity-client-id"- :display-name (deferred-tru "Client ID")- :required true- :visible-if {"auth-provider" "azure-managed-identity"}}- {:name "oauth-token-url"- :display-name (deferred-tru "Auth token URL")- :required true- :visible-if {"auth-provider" "oauth"}}- {:name "oauth-token-headers"- :display-name (deferred-tru "Auth token request headers (a JSON map)")- :visible-if {"auth-provider" "oauth"}}])+(defn auth-provider-options+ "Options for using an auth provider instead of a literal password.+ When called with no arguments, returns options for all available auth providers.+ When called with a collection of provider keywords (e.g., #{:aws-iam}), returns options+ filtered to only those providers."+ ([]+ (auth-provider-options nil))+ ([allowed-providers]+ (let [all-provider-options [{:name (deferred-tru "Azure Managed Identity")+ :value "azure-managed-identity"}+ {:name (deferred-tru "AWS IAM")+ :value "aws-iam"}+ {:name (deferred-tru "OAuth")+ :value "oauth"}]+ provider-options (if (seq allowed-providers)+ (let [allowed-set (set (map name allowed-providers))]+ (filterv #(contains? allowed-set (:value %)) all-provider-options))+ all-provider-options)+ default-provider (:value (first provider-options))]+ [{:name "use-auth-provider"+ :type :checked-section+ :check (fn []+ (and+ ;; Managed Identities only make sense if Metabase is in the same cloud as the DW+ (not (premium-features/is-hosted?))+ (premium-features/enable-database-auth-providers?)))+ :default false}+ {:name "auth-provider"+ :display-name (deferred-tru "Auth provider")+ :type :select+ :options provider-options+ :default default-provider+ :visible-if {"use-auth-provider" true}}+ {:name "azure-managed-identity-client-id"+ :display-name (deferred-tru "Client ID")+ :required true+ :visible-if {"auth-provider" "azure-managed-identity"}}+ {:name "oauth-token-url"+ :display-name (deferred-tru "Auth token URL")+ :required true+ :visible-if {"auth-provider" "oauth"}}+ {:name "oauth-token-headers"+ :display-name (deferred-tru "Auth token request headers (a JSON map)")+ :visible-if {"auth-provider" "oauth"}}])));;; +----------------------------------------------------------------------------------------------------------------+;;; | Class -> Base Type |diff --git a/src/metabase/driver/mysql.clj b/src/metabase/driver/mysql.cljindex 834e7ef0a79..037be11fb06 100644--- a/src/metabase/driver/mysql.clj+++ b/src/metabase/driver/mysql.clj@@ -227,7 +227,9 @@(assoc driver.common/default-port-details :placeholder 3306)driver.common/default-dbname-detailsdriver.common/default-user-details- driver.common/default-password-details+ (driver.common/auth-provider-options #{:aws-iam})+ (assoc driver.common/default-password-details+ :visible-if {"use-auth-provider" false})driver.common/default-role-detailsdriver.common/cloud-ip-address-infodriver.common/default-ssl-details@@ -651,23 +653,42 @@(set-prog-nm-fn)))) ; additional-options did not contain connectionAttributes at all; set it(defmethod sql-jdbc.conn/connection-details->spec :mysql- [_ {ssl? :ssl, :keys [additional-options ssl-cert], :as details}]+ [_ {ssl? :ssl, :keys [additional-options ssl-cert auth-provider], :as details}];; In versions older than 0.32.0 the MySQL driver did not correctly save `ssl?` connection status. Users worked;; around this by including `useSSL=true`. Check if that's there, and if it is, assume SSL status. See #9629;;;; TODO - should this be fixed by a data migration instead?(let [addl-opts-map (sql-jdbc.common/additional-options->map additional-options :url "=" false)+ use-iam? (= (some-> auth-provider keyword) :aws-iam)ssl? (or ssl? (= "true" (get addl-opts-map "useSSL")))ssl-cert? (and ssl? (some? ssl-cert))](when (and ssl? (not (contains? addl-opts-map "trustServerCertificate")))(log/info "You may need to add 'trustServerCertificate=true' to the additional connection options to connect with SSL."))+ (when (and use-iam? (not ssl?))+ (throw (ex-info "You must enable SSL in order to use AWS IAM authentication" {})))+ (when (and use-iam?+ (contains? addl-opts-map "sslMode")+ (not (contains? addl-opts-map "sslMode=VERIFY_CA")))+ (throw (ex-info "sslMode must be VERIFY_CA in order to use AWS IAM authentication" {})))(mergedefault-connection-args;; newer versions of MySQL will complain if you don't specify this when not using SSL{:useSSL (boolean ssl?)}- (let [details (-> (if ssl-cert? (set/rename-keys details {:ssl-cert :serverSslCert}) details)- (set/rename-keys {:dbname :db})- (dissoc :ssl))]+ (let [details (cond-> details+ ssl-cert?+ (set/rename-keys {:ssl-cert :serverSslCert})++ use-iam?+ (->+ (assoc :subprotocol "aws-wrapper:mysql"+ :classname "software.amazon.jdbc.ds.AwsWrapperDataSource"+ :sslMode "VERIFY_CA"+ :wrapperPlugins "iam")+ (dissoc :auth-provider :use-auth-provider))++ true+ (-> (set/rename-keys {:dbname :db})+ (dissoc :ssl)))](-> (driver-api/spec :mysql details)(maybe-add-program-name-option addl-opts-map)(sql-jdbc.common/handle-additional-options details))))))diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.cljindex 947c4846043..86e738364c8 100644--- a/src/metabase/driver/postgres.clj+++ b/src/metabase/driver/postgres.clj@@ -158,7 +158,7 @@(assoc driver.common/default-port-details :placeholder 5432)driver.common/default-dbname-detailsdriver.common/default-user-details- driver.common/auth-provider-options+ (driver.common/auth-provider-options)(assoc driver.common/default-password-details:visible-if {"use-auth-provider" false})driver.common/cloud-ip-address-info@@ -920,8 +920,9 @@{:sslmode "disable"})(defmethod sql-jdbc.conn/connection-details->spec :postgres- [_ {ssl? :ssl, :as details-map}]- (let [props (-> details-map+ [_ {ssl? :ssl, :keys [auth-provider], :as details-map}]+ (let [use-iam? (= (some-> auth-provider keyword) :aws-iam)+ props (-> details-map(update :port (fn [port](if (string? port)(Integer/parseInt port)@@ -937,10 +938,19 @@;; internal property values back; only merge in the ones the driver might recognize(merge ssl-prms (select-keys props (keys ssl-prms))))(merge disable-ssl-params props))+ props (if use-iam?+ (-> props+ (assoc :subprotocol "aws-wrapper:postgresql"+ :classname "software.amazon.jdbc.ds.AwsWrapperDataSource"+ :wrapperPlugins "iam")+ (dissoc :auth-provider :use-auth-provider))+ props)props (as-> props it(set/rename-keys it {:dbname :db})(driver-api/spec :postgres it)(sql-jdbc.common/handle-additional-options it details-map))]+ (when (and use-iam? (not ssl?))+ (throw (ex-info "You must enable SSL in order to use AWS IAM authentication" {})))props))(defmethod sql-jdbc.sync/excluded-schemas :postgres [_driver] #{"information_schema" "pg_catalog"})diff --git a/src/metabase/driver/sql_jdbc/common.clj b/src/metabase/driver/sql_jdbc/common.cljindex 2c2bcede538..b543079dbe3 100644--- a/src/metabase/driver/sql_jdbc/common.clj+++ b/src/metabase/driver/sql_jdbc/common.clj@@ -43,7 +43,7 @@v)) "" additional-opts)))(defn handle-additional-options- "If `details` contains an `:addtional-options` key, append those options to the connection string in `connection-spec`.+ "If `details` contains an `:additional-options` key, append those options to the connection string in `connection-spec`.(Some drivers like MySQL provide this details field to allow special behavior where needed).Optionally specify `seperator-style`, which defaults to `:url` (e.g. `?a=1&b=2`). You may instead set it todiff --git a/test/metabase/app_db/aws_iam_test.clj b/test/metabase/app_db/aws_iam_test.cljnew file mode 100644index 00000000000..ac1ebf81f2d--- /dev/null+++ b/test/metabase/app_db/aws_iam_test.clj@@ -0,0 +1,114 @@+(ns metabase.app-db.aws-iam-test+ "Integration tests for AWS IAM authentication with application database.++ These tests require actual AWS RDS instances configured for IAM authentication.+ Tests will be skipped if required environment variables are not set.++ Required environment variables:+ - Postgres:+ - MB_POSTGRES_AWS_IAM_TEST_HOST+ - MB_POSTGRES_AWS_IAM_TEST_PORT+ - MB_POSTGRES_AWS_IAM_TEST_USER+ - MB_POSTGRES_AWS_IAM_TEST_DBNAME+ - MySQL:+ - MB_MYSQL_AWS_IAM_TEST_HOST+ - MB_MYSQL_AWS_IAM_TEST_PORT+ - MB_MYSQL_AWS_IAM_TEST_USER+ - MB_MYSQL_AWS_IAM_TEST_DBNAME+ - MB_MYSQL_AWS_IAM_TEST_SSL_CERT (required, PEM certificate content/trust)"+ (:require+ [clojure.test :refer :all]+ [metabase.app-db.data-source :as mdb.data-source]+ [metabase.config.core :as config]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(defn- get-connection-and-close!+ "Attempt to get a connection from the DataSource and close it immediately.+ Returns true if successful, throws if connection fails."+ [^javax.sql.DataSource datasource]+ (with-open [conn (.getConnection datasource)]+ (assert conn "Connection should not be nil")+ true))++(deftest ^:mb/once postgres-aws-iam-test+ (if (config/config-bool :mb-postgres-aws-iam-test)+ (let [host (config/config-str :mb-postgres-aws-iam-test-host)+ port (config/config-int :mb-postgres-aws-iam-test-port)+ user (config/config-str :mb-postgres-aws-iam-test-user)+ dbname (config/config-str :mb-postgres-aws-iam-test-dbname)+ uri (format "postgres://%s:%d/%s?user=%s" host port dbname user)]+ (testing "Connection details are configured"+ (is (string? host))+ (is (string? user))+ (is (int? port))+ (is (string? dbname)))++ (testing "using broken-out details"+ (testing "Can create DataSource with AWS IAM enabled"+ (let [details {:host host+ :port port+ :user user+ :db dbname+ :aws-iam true}+ datasource (mdb.data-source/broken-out-details->DataSource :postgres details)]+ (is (instance? javax.sql.DataSource datasource))++ (testing "Can establish connection using AWS IAM"+ (is (true? (get-connection-and-close! datasource)))))))++ (testing "using uri"+ (testing "Can create DataSource with AWS IAM enabled"+ (let [datasource (mdb.data-source/raw-connection-string->DataSource+ uri nil nil nil true)]+ (is (instance? javax.sql.DataSource datasource))++ (testing "Can establish connection using AWS IAM"+ (is (true? (get-connection-and-close! datasource))))))))+ (log/info "Skipping test: MB_POSTGRES_AWS_IAM_TEST not set")))++(deftest ^:mb/once mysql-aws-iam-test+ (testing "MySQL App DB connection with AWS IAM authentication"+ (if (config/config-bool :mb-mysql-aws-iam-test)+ (let [host (config/config-str :mb-mysql-aws-iam-test-host)+ port (config/config-int :mb-mysql-aws-iam-test-port)+ user (config/config-str :mb-mysql-aws-iam-test-user)+ dbname (config/config-str :mb-mysql-aws-iam-test-dbname)+ ssl-cert (config/config-str :mb-mysql-aws-iam-test-ssl-cert)+ uri (format "mysql://%s:%d/%s?user=%s&%s" host port dbname user+ (if (= ssl-cert "trust")+ "trustServerCertificate=true"+ (str "?sslMode=VERIFY_CA&serverSslCert=" ssl-cert)))]+ (testing "Connection details are configured"+ (is (string? host))+ (is (string? user))+ (is (int? port))+ (is (string? dbname)))++ (testing "SSL certificate is configured"+ (is (string? ssl-cert)))++ (testing "using broken-out details"+ (testing "Can create DataSource with AWS IAM enabled"+ (let [details {:host host+ :port port+ :user user+ :db dbname+ :ssl-cert ssl-cert+ :aws-iam true}+ datasource (mdb.data-source/broken-out-details->DataSource :mysql details)]+ (is (instance? javax.sql.DataSource datasource))++ (testing "Can establish connection using AWS IAM"+ (is (true? (get-connection-and-close! datasource)))))))++ (testing "using uri"+ (testing "Can create DataSource with AWS IAM enabled"+ (let [datasource (mdb.data-source/raw-connection-string->DataSource+ uri nil nil nil true)]+ (is (instance? javax.sql.DataSource datasource))++ (testing "Can establish connection using AWS IAM"+ (is (true? (get-connection-and-close! datasource))))))))+ (log/info "Skipping test: MB_MYSQL_AWS_IAM_TEST not set"))))diff --git a/test/metabase/app_db/data_source_test.clj b/test/metabase/app_db/data_source_test.cljindex ada307cb8e4..89ebfc3f47a 100644--- a/test/metabase/app_db/data_source_test.clj+++ b/test/metabase/app_db/data_source_test.clj@@ -84,6 +84,70 @@(is (= [{:one 1}](jdbc/query {:connection conn} "SELECT 1 AS one;")))))))+(deftest ^:parallel broken-out-details-test-6+ (testing :aws-iam+ (testing "Postgres with AWS IAM"+ (is (= (->DataSource+ "jdbc:aws-wrapper:postgresql://localhost:5432/metabase"+ {"ApplicationName" config/mb-version-and-process-identifier+ "OpenSourceSubProtocolOverride" "true"+ "user" "cam"+ "useSSL" true+ "wrapperPlugins" "iam"})+ (mdb.data-source/broken-out-details->DataSource :postgres {:host "localhost"+ :port 5432+ :user "cam"+ :aws-iam true+ :db "metabase"}))))))++(deftest ^:parallel broken-out-details-test-7+ (testing :aws-iam+ (testing "MySQL with AWS IAM"+ (is (= (->DataSource+ "jdbc:aws-wrapper:mysql://localhost:3306/metabase"+ {"user" "root"+ "wrapperPlugins" "iam"+ "useSSL" true+ "sslMode" "VERIFY_CA"})+ (mdb.data-source/broken-out-details->DataSource :mysql {:host "localhost"+ :port 3306+ :user "root"+ :aws-iam true+ :db "metabase"}))))))++(deftest ^:parallel broken-out-details-test-8+ (testing :aws-iam+ (testing "MySQL with AWS IAM and ssl-cert=trust"+ (is (= (->DataSource+ "jdbc:aws-wrapper:mysql://localhost:3306/metabase"+ {"user" "root"+ "wrapperPlugins" "iam"+ "useSSL" true+ "trustServerCertificate" "true"})+ (mdb.data-source/broken-out-details->DataSource :mysql {:host "localhost"+ :port 3306+ :user "root"+ :aws-iam true+ :ssl-cert "trust"+ :db "metabase"}))))))++(deftest ^:parallel broken-out-details-test-9+ (testing :aws-iam+ (testing "MySQL with AWS IAM and ssl-cert path"+ (is (= (->DataSource+ "jdbc:aws-wrapper:mysql://localhost:3306/metabase"+ {"user" "root"+ "wrapperPlugins" "iam"+ "sslMode" "VERIFY_CA"+ "useSSL" true+ "serverSslCert" "/path/to/certificate.pem"})+ (mdb.data-source/broken-out-details->DataSource :mysql {:host "localhost"+ :port 3306+ :user "root"+ :aws-iam true+ :ssl-cert "/path/to/certificate.pem"+ :db "metabase"}))))))+(deftest ^:parallel connection-string-test(let [data-source (mdb.data-source/raw-connection-string->DataSource(format "jdbc:h2:mem:%s" (mt/random-name)))]@@ -169,6 +233,24 @@(mdb.data-source/raw-connection-string->DataSource "postgres://metabase" "" "1234" nil)(mdb.data-source/raw-connection-string->DataSource "postgres://metabase" "" "1234" "client ID"))))))+(deftest ^:parallel raw-connection-string-with-aws-iam-test+ (testing "Raw connection string with AWS IAM enabled for Postgres"+ (is (= (->DataSource+ "jdbc:aws-wrapper:postgresql://metabase"+ {"user" "cam"+ "useSSL" true+ "wrapperPlugins" "iam"})+ (mdb.data-source/raw-connection-string->DataSource "postgres://metabase" "cam" nil nil true)))))++(deftest ^:parallel raw-connection-string-with-aws-iam-test-2+ (testing "Raw connection string with AWS IAM enabled for MySQL"+ (is (= (->DataSource+ "jdbc:aws-wrapper:mysql://metabase"+ {"user" "cam"+ "useSSL" true+ "wrapperPlugins" "iam"})+ (mdb.data-source/raw-connection-string->DataSource "mysql://metabase" "cam" nil nil true)))))+(deftest ^:parallel equality-test(testing "Two DataSources with the same URL should be equal"(is (= (mdb.data-source/raw-connection-string->DataSource "ABCD")diff --git a/test/metabase/driver/sql_jdbc/connection_test.clj b/test/metabase/driver/sql_jdbc/connection_test.cljindex 3489cec120b..0caea3630b0 100644--- a/test/metabase/driver/sql_jdbc/connection_test.clj+++ b/test/metabase/driver/sql_jdbc/connection_test.clj@@ -18,6 +18,7 @@[metabase.driver.sql-jdbc.connection.ssh-tunnel-test :as ssh-test][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.util :as driver.u]+ [metabase.premium-features.core :as premium-features][metabase.query-processor :as qp][metabase.query-processor.test-util :as qp.test-util][metabase.sync.core :as sync]@@ -28,6 +29,7 @@[metabase.test.util :as tu][metabase.util :as u][metabase.util.http :as u.http]+ [metabase.util.log :as log][next.jdbc :as next.jdbc][toucan2.core :as t2])(:import@@ -358,6 +360,63 @@;; we must have created more than one connection(is (> @connection-creations 1))))))))))+#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}+(deftest test-aws-iam-auth-provider-connection+ (mt/with-premium-features #{:database-auth-providers}+ (testing "AWS IAM authentication for Postgres"+ (mt/test-driver :postgres+ (let [db-details (:details (mt/db))+ iam-db-details (-> db-details+ (dissoc :password)+ (assoc :use-auth-provider true+ :auth-provider :aws-iam+ :ssl true))]+ (testing "Connection spec is configured with AWS wrapper"+ (let [spec (sql-jdbc.conn/connection-details->spec :postgres iam-db-details)]+ (is (= "aws-wrapper:postgresql" (:subprotocol spec)))+ (is (= "software.amazon.jdbc.ds.AwsWrapperDataSource" (:classname spec)))+ (is (= "iam" (:wrapperPlugins spec))))))))+ (testing "AWS IAM authentication for MySQL"+ (mt/test-driver :mysql+ (let [db-details (:details (mt/db))+ iam-db-details (-> db-details+ (dissoc :password)+ (assoc :use-auth-provider true+ :auth-provider :aws-iam+ :ssl true))]+ (testing "Connection spec is configured with AWS wrapper"+ (let [spec (sql-jdbc.conn/connection-details->spec :mysql iam-db-details)]+ (is (= "aws-wrapper:mysql" (:subprotocol spec)))+ (is (= "software.amazon.jdbc.ds.AwsWrapperDataSource" (:classname spec)))+ (is (= "iam" (:wrapperPlugins spec)))+ (is (= "VERIFY_CA" (:sslMode spec))))))))))++#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}+(deftest ^:parallel test-aws-iam-requires-ssl+ (testing "AWS IAM authentication requires SSL to be enabled"+ (testing "Postgres throws error when SSL is disabled"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"You must enable SSL in order to use AWS IAM authentication"+ (sql-jdbc.conn/connection-details->spec :postgres+ {:host "localhost"+ :port 5432+ :user "cam"+ :auth-provider :aws-iam+ :ssl false+ :db "metabase"}))))+ (testing "MySQL throws error when SSL is disabled"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"You must enable SSL in order to use AWS IAM authentication"+ (sql-jdbc.conn/connection-details->spec :mysql+ {:host "localhost"+ :port 3306+ :user "root"+ :auth-provider :aws-iam+ :ssl false+ :db "metabase"}))))))+(defmacro ^:private with-tunnel-details![& body]`(let [original-details# (:details (mt/db))@@ -508,3 +567,58 @@;; check the query again; the tunnel should have been reestablished(check-data))))(finally (.stop ^Server server))))))))++#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}+(deftest postgres-aws-iam-can-connect+ (if (config/config-bool :mb-postgres-aws-iam-test)+ (let [host (config/config-str :mb-postgres-aws-iam-test-host)+ port (config/config-int :mb-postgres-aws-iam-test-port)+ user (config/config-str :mb-postgres-aws-iam-test-user)+ dbname (config/config-str :mb-postgres-aws-iam-test-dbname)]+ (with-redefs [premium-features/is-hosted? (constantly false)]+ (testing "Connection details are configured"+ (is (string? host))+ (is (string? user))+ (is (int? port))+ (is (string? dbname)))++ (mt/with-temporary-setting-values [db-connection-timeout-ms 10000]+ (is+ (driver.u/can-connect-with-details? :postgres {:host host+ :port port+ :dbname dbname+ :user user+ :use-auth-provider true+ :auth-provider :aws-iam+ :ssl true})))))+ (log/info "Skipping test: MB_POSTGRES_AWS_IAM_TEST not set")))++#_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}+(deftest mysql-aws-iam-can-connect+ (if (config/config-bool :mb-mysql-aws-iam-test)+ (let [host (config/config-str :mb-mysql-aws-iam-test-host)+ port (config/config-int :mb-mysql-aws-iam-test-port)+ user (config/config-str :mb-mysql-aws-iam-test-user)+ dbname (config/config-str :mb-mysql-aws-iam-test-dbname)+ ssl-cert (config/config-str :mb-mysql-aws-iam-test-ssl-cert)]+ (with-redefs [premium-features/is-hosted? (constantly false)]+ (testing "Connection details are configured"+ (is (string? host))+ (is (string? user))+ (is (int? port))+ (is (string? dbname))+ (is (string? ssl-cert)))++ (mt/with-temporary-setting-values [db-connection-timeout-ms 10000]+ (is+ (driver.u/can-connect-with-details? :mysql {:host host+ :port port+ :dbname dbname+ :user user+ :additional-options (if (= ssl-cert "trust")+ "trustServerCertificate=true"+ (str "serverSslCert=" ssl-cert))+ :use-auth-provider true+ :auth-provider :aws-iam+ :ssl true})))))+ (log/info "Skipping test: MB_MYSQL_AWS_IAM_TEST not set")))