The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve oracle parsing (#80043)
Nathan Voxlanda25cc1f7be16
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 0643979203b..09214e8e1f4 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -191,6 +191,8 @@metabase.channel.render.pdf.font/load-fonts!metabase.warehouses.models.database/assert-not-h2!metabase.driver.sql-jdbc.execute/execute-prepared-statement!+ ;; only inspects a details map and throws; the `!` is for the throw, not for a mutation+ metabase.driver/validate-db-details!metabase.mcp.usage/record-mcp-session!metabase.mcp.usage/record-mcp-session-end!metabase.mcp.usage/record-mcp-tool-call!diff --git a/modules/drivers/oracle/src/metabase/driver/oracle.clj b/modules/drivers/oracle/src/metabase/driver/oracle.cljindex c881db2ff54..4560465f7cd 100644--- a/modules/drivers/oracle/src/metabase/driver/oracle.clj+++ b/modules/drivers/oracle/src/metabase/driver/oracle.clj@@ -1,5 +1,5 @@(ns metabase.driver.oracle- (:refer-clojure :exclude [mapv])+ (:refer-clojure :exclude [mapv not-empty select-keys])(:require[clojure.java.jdbc :as jdbc][clojure.string :as str]@@ -11,7 +11,6 @@[metabase.driver.impl :as driver.impl][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc :as sql-jdbc]- [metabase.driver.sql-jdbc.common :as sql-jdbc.common][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]@@ -25,10 +24,11 @@[metabase.driver.sql.util :as sql.u][metabase.util.date-2 :as u.date][metabase.util.honey-sql-2 :as h2x]+ [metabase.util.i18n :refer [deferred-tru tru]][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.performance :refer [mapv]])+ [metabase.util.performance :refer [mapv not-empty select-keys]])(:import(com.mchange.v2.c3p0 C3P0ProxyConnection)(java.security KeyStore)@@ -48,6 +48,15 @@(set! *warn-on-reflection* true)+;; Turn off Oracle's "centralized configuration providers", which are enabled by default and which read a connect+;; descriptor as instructions rather than as an address: `@config-https://host/path` makes the driver fetch that URL+;; and obey the configuration it returns, and `@config-file:///path` makes it read that file.++;; `(NONE)` is the driver's spelling for "no providers at all". `oracle.jdbc.driver.OracleDriver` reads this in its+;; static initializer, so it has to be set before that class is initialized: the plugin manifest loads this namespace+;; before the `register-jdbc-driver` step that triggers it.+(System/setProperty "oracle.jdbc.configurationProviders" "(NONE)")+(driver/register! :oracle, :parent #{:sql-jdbc ::sql.qp.empty-string-is-null/empty-string-is-null})(defmethod driver/host-carrying-parameters :oracle@@ -104,6 +113,83 @@[:ssl-truststore-options {:optional true} [:maybe string?]][:ssl-truststore-password-value {:optional true} [:maybe string?]]])+;;; Everything after the `@` in `jdbc:oracle:thin:@...` is a *connect descriptor*, which the driver reads as+;;; instructions and not merely as an address: a scheme there sends it to fetch a URL, read a file, or ask an LDAP+;;; directory where to connect, and the `(KEY=value)` syntax lets a value that escapes its own clause name a different+;;; host entirely. Every detail below is concatenated into that descriptor, so each is allowlisted down to a plain+;;; value first. An allowlist rather than a denylist of the dangerous forms: the punctuation that gives the descriptor+;;; its structure -- `:` `/` `(` `)` `=` `?` `&` `@` `,` -- has no place in a hostname, a port, a SID, or a service+;;; name, and leaving any of it through is what makes `config-https://attacker/x` or `x))(ADDRESS=(HOST=attacker)` a+;;; connection Metabase would open.++(def ^:private detail-patterns+ "What each detail interpolated into the connect descriptor is allowed to look like, keyed by detail."+ {;; a hostname or IPv4 literal, or an IPv6 literal in the brackets Oracle needs to tell it from the port separator+ :host #"[A-Za-z0-9][A-Za-z0-9._-]*|\[[0-9A-Fa-f:.]+\]"+ ;; bounded rather than `\d+` so that the `parse-long` in [[validated-details]] is total -- no TCP port needs more+ :port #"\d{1,5}"+ ;; `$` and `#` are legal in an Oracle identifier; a service name is commonly qualified, e.g.+ ;; `db_high.adb.oraclecloud.com`+ :sid #"[A-Za-z0-9][A-Za-z0-9._$#-]*"+ :service-name #"[A-Za-z0-9][A-Za-z0-9._$#-]*"})++(def ^:private detail-display-names+ "How the connection form labels each detail, so an error names the field the user can actually see."+ {:host (deferred-tru "host")+ :port (deferred-tru "port")+ :sid (deferred-tru "Oracle system ID (SID)")+ :service-name (deferred-tru "Oracle service name")})++(defn- invalid-detail-exception [detail-key]+ (let [message (tru "Invalid {0}: it must be a plain value, not a URL or an Oracle connect descriptor."+ (detail-display-names detail-key))]+ (ex-info message {:status-code 400, :message message})))++(defn- validated-details+ "`details` with every value that reaches the connect descriptor reduced to its canonical form: strings trimmed, the+ port read as a number, and blanks dropped so the caller's defaults apply. Throws an `ex-info` when one of them is not+ a plain value of the kind it claims to be. Build the descriptor from the map this returns, so that no value is+ checked in one form and used in another."+ [details]+ (reduce-kv (fn [details detail-key pattern]+ ;; surrounding whitespace is a copy-and-paste artifact rather than an attempt at anything, so it is+ ;; trimmed away instead of being grounds for refusal+ (if-let [value (some-> (detail-key details) str str/trim not-empty)]+ (do+ (when-not (re-matches pattern value)+ (throw (invalid-detail-exception detail-key)))+ (assoc details detail-key (cond-> value (= detail-key :port) parse-long)))+ ;; a blank detail names nowhere. Dropping it rather than passing `""` along lets the `:or` defaults in+ ;; [[sql-jdbc.conn/connection-details->spec]] apply, and keeps a blank port out of [[ssl-spec]]'s `%d`+ (dissoc details detail-key)))+ details+ detail-patterns))++(defmethod driver/validate-db-details! :oracle+ [_driver details]+ (validated-details details)+ ;; Oracle's connection form has never offered an `additional-options` field, and the one place the driver used to+ ;; honor it -- the SSL branch -- turned it into `?name=value` parameters on the descriptor, which is a way around+ ;; [[connection-property-keys]] below. Warned about here rather than in `connection-details->spec`, which runs for+ ;; every connection pool and every host check; this runs when an admin saves or tests the database.+ (when-not (str/blank? (:additional-options details))+ (log/warn "Ignoring additional-options on an Oracle database: this driver does not support extra JDBC options.")))++(def ^:private connection-property-keys+ "The only detail keys that become JDBC connection properties. Everything else a details map carries -- the SSH tunnel+ keys, the secret bookkeeping, the scheduling options, and anything an API caller invented -- belongs to Metabase, not+ to the Oracle client."+ [:user+ :password+ ;; written by handle-ssl-options, never by a user+ :javax.net.ssl.keyStore+ :javax.net.ssl.keyStorePassword+ :javax.net.ssl.keyStoreType+ :javax.net.ssl.trustStore+ :javax.net.ssl.trustStorePassword+ :javax.net.ssl.trustStoreType+ :oracle.net.authentication_services])+(defmethod driver/prettify-native-form :oracle[_ native-form](sql.u/format-sql-and-fix-params :plsql native-form))@@ -149,7 +235,13 @@[_ column-type](database-type->base-type column-type))-(mu/defn- non-ssl-spec [_details :- ::details spec host port sid service-name]+;;; both take the values [[validated-details]] returned, which is what makes it safe to concatenate them+(mu/defn- non-ssl-spec :- :map+ [spec :- :map+ host :- :string+ port :- :int+ sid :- [:maybe :string]+ service-name :- [:maybe :string]](assoc spec :subname (str "@" host":" port(when sid@@ -157,13 +249,17 @@(when service-name(str "/" service-name)))))-(mu/defn- ssl-spec [details :- ::details spec host port sid service-name]- (-> (assoc spec :subname (format "@(DESCRIPTION=(ADDRESS=(PROTOCOL=tcps)(HOST=%s)(PORT=%d))(CONNECT_DATA=%s%s))"- host- port- (if sid (str "(SID=" sid ")") "")- (if service-name (str "(SERVICE_NAME=" service-name ")") "")))- (sql-jdbc.common/handle-additional-options details)))+(mu/defn- ssl-spec :- :map+ [spec :- :map+ host :- :string+ port :- :int+ sid :- [:maybe :string]+ service-name :- [:maybe :string]]+ (assoc spec :subname (format "@(DESCRIPTION=(ADDRESS=(PROTOCOL=tcps)(HOST=%s)(PORT=%d))(CONNECT_DATA=%s%s))"+ host+ port+ (if sid (str "(SID=" sid ")") "")+ (if service-name (str "(SERVICE_NAME=" service-name ")") ""))))(def ^:private ^:const prog-name-property"The connection property used by the Oracle JDBC Thin Driver to control the program name."@@ -207,26 +303,30 @@details))(mu/defmethod sql-jdbc.conn/connection-details->spec :oracle- [_driver- {:keys [host port sid service-name]- :or {host "localhost", port 1521}- :as details} :- ::details]- (assert (or sid service-name))- (let [spec {:classname "oracle.jdbc.OracleDriver", :subprotocol "oracle:thin"}- finish-fn (partial (if (:ssl details) ssl-spec non-ssl-spec) details)+ [_driver details :- ::details]+ ;; validated here rather than only in [[driver/validate-db-details!]] so that no connection path -- sync, a query, a+ ;; pool being rebuilt from details saved before this check existed -- can reach the driver without it+ (let [{:keys [host port sid service-name]+ :or {host "localhost", port 1521}+ :as details} (validated-details details)+ _ (assert (or sid service-name))+ spec {:classname "oracle.jdbc.OracleDriver", :subprotocol "oracle:thin"}+ finish-fn (if (:ssl details) ssl-spec non-ssl-spec);; the v$session.program value has a max length of 48 (see T4Connection), so we have to make it more terse than;; the usual config/mb-version-and-process-identifier string and ensure we truncate to a length of 48prog-nm (as-> (format "MB %s %s" (driver-api/mb-version-info :tag) driver-api/local-process-uuid) s(subs s 0 (min 48 (count s))))]- (-> (merge spec details)+ ;; an allowlist, not the whole details map: an extra key such as `oracle.net.tns_admin` or+ ;; `oracle.net.httpsProxyHost` would otherwise reach the driver as a connection property and make it read a local+ ;; file, or route its egress through a host of the caller's choosing+ (-> (merge spec (select-keys (handle-ssl-options details) connection-property-keys))(assoc prog-name-property prog-nm)- handle-ssl-options- (dissoc :host :port :sid :service-name :ssl)(finish-fn host port sid service-name))))(mu/defmethod driver/can-connect? :oracle[driverdetails :- ::details]+ (driver/validate-db-details! driver details)(sql-jdbc.conn/with-connection-spec-for-testing-connection [jdbc-spec [driver details]](= 1M (first (vals (first (jdbc/query jdbc-spec ["SELECT 1 FROM dual"])))))))diff --git a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj b/modules/drivers/oracle/test/metabase/driver/oracle_test.cljindex 8aa3baab134..aa4d485f2f3 100644--- a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj+++ b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj@@ -40,6 +40,8 @@[metabase.util.date-2 :as u.date][metabase.util.honey-sql-2 :as h2x][metabase.util.log :as log]+ [metabase.util.log.capture :as log.capture]+ [metabase.util.malli :as mu][toucan2.core :as t2])(:import(java.time LocalDateTime)))@@ -110,6 +112,162 @@(catch Throwable e(driver/humanize-connection-error-message :oracle (u/all-ex-messages e))))))))+;;; The Oracle thin driver reads its connect descriptor -- everything after the `@` -- as instructions, not+;;; just as an address: `@config-https://host/path` makes it fetch that URL and obey the configuration it returns,+;;; `@config-file:///path` makes it read that file, and `@ldap://host/...` makes it ask that host for the real+;;; descriptor. The details below are all built by concatenation into that descriptor, so each one has to be what it+;;; claims to be before it gets there.+(deftest ^:parallel reject-connect-descriptor-injection-test+ (doseq [[message details] [["a host naming Oracle's HTTPS configuration provider (SSRF)"+ {:host "config-https://evil.example.com/c", :sid "ORCL"}]+ ["a host naming Oracle's file configuration provider (local file read)"+ {:host "config-file:///etc/passwd", :sid "ORCL"}]+ ["a host naming an LDAP directory (SSRF + connection redirection)"+ {:host "ldap://evil.example.com/cn=x,cn=OracleContext", :sid "ORCL"}]+ ["a host carrying any other scheme"+ {:host "tcps://evil.example.com", :sid "ORCL"}]+ ["a host carrying descriptor syntax"+ {:host "h)(PORT=1521))(ADDRESS=(HOST=evil", :sid "ORCL", :ssl true}]+ ["a SID that closes the CONNECT_DATA clause and adds an address"+ {:host "h", :sid "x))(ADDRESS=(PROTOCOL=tcp)(HOST=evil)(PORT=80)", :ssl true}]+ ["a service name that closes the CONNECT_DATA clause and adds an address"+ {:host "h", :service-name "x))(ADDRESS=(PROTOCOL=tcp)(HOST=evil)(PORT=80)", :ssl true}]+ ["a service name carrying connection parameters"+ {:host "h", :service-name "svc?oracle.net.tns_admin=/etc"}]]]+ (testing message+ (testing "is refused when the details are validated"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"(?i)invalid"+ (driver/validate-db-details! :oracle details))))+ (testing "is refused when a connection spec is built, so no connection path can skip the check"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"(?i)invalid"+ (sql-jdbc.conn/connection-details->spec :oracle details)))))))++;;; `::details` types the port as an integer, but that schema is instrumented only in dev and test, so in production a+;;; string port arrives as-is. These two go through `driver/validate-db-details!`, which is not schema-gated, so they+;;; exercise what production sees rather than what the schema would have caught first.+(deftest ^:parallel reject-port-carrying-connection-parameters-test+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"(?i)invalid"+ (driver/validate-db-details! :oracle {:host "h", :port "1521 ?oracle.net.tns_admin=/etc", :sid "ORCL"}))))++(deftest ^:parallel accept-port-sent-as-a-string-test+ (is (nil? (driver/validate-db-details! :oracle {:host "h", :port "1521", :sid "ORCL"}))))++(deftest ^:parallel accept-ordinary-connection-details-test+ (doseq [[message details] [["a hostname" {:host "oracle.example.com", :sid "ORCL"}]+ ["an IPv4 literal" {:host "10.0.0.4", :sid "ORCL"}]+ ["an IPv6 literal" {:host "[2001:db8::1]", :sid "ORCL"}]+ ["an underscore in the hostname" {:host "my_db.internal", :sid "ORCL"}]+ ["a missing host" {:sid "ORCL"}]+ ["a blank host" {:host "", :sid "ORCL"}]+ ["a qualified service name"+ {:host "h", :service-name "g1234_mydb_high.adb.oraclecloud.com"}]+ ["a SID with the punctuation Oracle allows in an identifier"+ {:host "h", :sid "MY_DB$1#x"}]]]+ (testing message+ (is (nil? (driver/validate-db-details! :oracle details)))+ (is (map? (sql-jdbc.conn/connection-details->spec :oracle details))))))++(deftest ^:parallel descriptor-details-are-normalized-test+ (testing "the value that was checked is the value the descriptor gets built from"+ (testing "surrounding whitespace is trimmed rather than passed through"+ (is (= "@localhost:1521:ORCL/MyService"+ (:subname (sql-jdbc.conn/connection-details->spec+ :oracle+ {:host " localhost ", :port 1521, :sid " ORCL ", :service-name " MyService "})))))+ (testing "a port sent as a string reaches the SSL descriptor as a number"+ ;; `::details` types the port as an integer, but that schema is instrumented only in dev and test, so in+ ;; production an API caller's string arrives as-is -- and `ssl-spec` formats the port with `%d`. Enforcement is+ ;; disabled here to exercise what production does.+ (is (= "@(DESCRIPTION=(ADDRESS=(PROTOCOL=tcps)(HOST=h)(PORT=1521))(CONNECT_DATA=(SID=ORCL)))"+ (mu/disable-enforcement+ (:subname (sql-jdbc.conn/connection-details->spec+ :oracle+ {:host "h", :port "1521", :sid "ORCL", :ssl true}))))))+ (testing "a blank detail names nowhere, so the defaults apply rather than an empty string reaching the descriptor"+ (is (= "@(DESCRIPTION=(ADDRESS=(PROTOCOL=tcps)(HOST=localhost)(PORT=1521))(CONNECT_DATA=(SID=ORCL)))"+ (mu/disable-enforcement+ (:subname (sql-jdbc.conn/connection-details->spec+ :oracle+ {:host " ", :port "", :sid "ORCL", :ssl true}))))))))++(deftest ^:parallel additional-options-warning-test+ (let [details {:host "h", :port 1521, :sid "ORCL", :additional-options "oracle.net.tns_admin=/etc"}]+ (testing "validating details warns that additional-options is ignored, so an admin saving one hears about it"+ (log.capture/with-log-messages-for-level [messages [metabase.driver.oracle :warn]]+ (driver/validate-db-details! :oracle details)+ (is (=? [{:level :warn, :message #"(?i).*additional-options.*"}]+ (messages)))))+ (testing "building a connection spec does not, since that runs for every pool and every host check"+ (log.capture/with-log-messages-for-level [messages [metabase.driver.oracle :warn]]+ (sql-jdbc.conn/connection-details->spec :oracle details)+ (is (= [] (messages)))))))++(deftest ^:parallel connection-property-allowlist-test+ (testing "only allowlisted details become JDBC connection properties"+ (testing "an extra detail key is not handed to the driver as a property"+ ;; `oracle.net.tns_admin` makes the driver read files under that directory; `oracle.net.httpsProxyHost` routes+ ;; its egress through a host of the attacker's choosing.+ (let [spec (sql-jdbc.conn/connection-details->spec+ :oracle+ {:host "h", :port 1521, :sid "ORCL", :user "u", :password "p"+ (keyword "oracle.net.tns_admin") "/etc"+ (keyword "oracle.net.httpsProxyHost") "evil.example.com"+ (keyword "oracle.net.ssl_server_dn_match") "false"})]+ (is (= #{:classname :subprotocol :subname :user :password (deref #'oracle/prog-name-property)}+ (set (keys spec))))))+ (testing "nor when the extra key arrives as a string rather than a keyword"+ ;; `ms/KeywordizedMap` normalizes details keys on API decode, but the allowlist is what this relies on: it keeps+ ;; named keys only, so a string key is dropped whichever way the decoding goes.+ (let [spec (sql-jdbc.conn/connection-details->spec+ :oracle+ {:host "h", :port 1521, :sid "ORCL"+ "oracle.net.tns_admin" "/etc"+ "user" "not-the-user"})]+ (is (= #{:classname :subprotocol :subname (deref #'oracle/prog-name-property)}+ (set (keys spec))))))+ (testing "Metabase's own bookkeeping details are not handed to the driver either"+ (let [spec (sql-jdbc.conn/connection-details->spec+ :oracle+ {:host "h", :port 1521, :sid "ORCL"+ :tunnel-enabled true, :tunnel-host "bastion.example.com", :tunnel-user "tunnel"+ :advanced-options true, :auto-run-queries true, :let-user-control-scheduling false})]+ (is (= #{:classname :subprotocol :subname (deref #'oracle/prog-name-property)}+ (set (keys spec))))))+ (testing "the SSL truststore properties the driver does need still get through"+ (let [spec (sql-jdbc.conn/connection-details->spec+ :oracle+ {:host "h", :port 1521, :sid "ORCL", :ssl true+ :ssl-use-truststore true+ :ssl-truststore-options "uploaded"+ :ssl-truststore-value "data:application/octet-stream;base64,eA=="+ :ssl-truststore-password-value "pw"})]+ (is (= #{:classname :subprotocol :subname (deref #'oracle/prog-name-property)+ :javax.net.ssl.trustStore :javax.net.ssl.trustStoreType :javax.net.ssl.trustStorePassword}+ (set (keys spec))))))))++(deftest ^:parallel ignore-additional-options-test+ (testing "`additional-options` is not appended to the connect descriptor"+ ;; Oracle's connection form offers no `additional-options` field, and the only place the driver honored one was+ ;; the SSL branch, where it became a `?name=value` parameter on the descriptor -- a way around the property+ ;; allowlist above.+ (is (= "@(DESCRIPTION=(ADDRESS=(PROTOCOL=tcps)(HOST=h)(PORT=1521))(CONNECT_DATA=(SID=ORCL)))"+ (:subname (sql-jdbc.conn/connection-details->spec+ :oracle+ {:host "h", :port 1521, :sid "ORCL", :ssl true+ :additional-options "oracle.net.tns_admin=/etc"}))))))++(deftest ^:parallel configuration-providers-disabled-test+ (testing "Oracle's centralized configuration providers are turned off for the JVM"+ ;; They are enabled by default, and turn `@config-https://…` / `@config-file://…` descriptors into an outbound+ ;; fetch or a local file read. `(NONE)` is the driver's spelling for "no providers at all".+ (is (= "(NONE)" (System/getProperty "oracle.jdbc.configurationProviders")))))+(deftest connection-properties-test(testing "Connection properties should be returned properly (including transformation of secret types)"(with-redefs [premium-features/is-hosted? (constantly false)]