The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Drivers on classpath (#73519)
dpsutton823c6f69ae0a
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 5da604f5647..4f4bb54cbe2 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1516,7 +1516,6 @@{:team "DevEx":api #{metabase.plugins.core}:uses #{classloader- configdriverutil}}diff --git a/bin/build/src/build.clj b/bin/build/src/build.cljindex a22c709f0a3..e71bae38036 100644--- a/bin/build/src/build.clj+++ b/bin/build/src/build.clj@@ -1,6 +1,5 @@(ns build(:require- [build-drivers :as build-drivers][build.licenses :as license][build.python :as build.python][build.uberjar :as uberjar]@@ -89,8 +88,8 @@(build-frontend! edition)):licenses (fn [{:keys [edition]}](build-licenses! edition))- :drivers (fn [{:keys [edition]}]- (build-drivers/build-drivers! edition))+ #_#_:drivers (fn [{:keys [edition]}]+ (build-drivers/build-drivers! edition)):python (fn [{:keys [edition]}](build.python/build-python-deps! edition)):uberjar (fn [{:keys [edition]}]diff --git a/bin/build/src/build/uberjar.clj b/bin/build/src/build/uberjar.cljindex 0d7f2fe0a5b..b87d653ab3a 100644--- a/bin/build/src/build/uberjar.clj+++ b/bin/build/src/build/uberjar.clj@@ -9,7 +9,7 @@[metabuild-common.core :as u][org.corfield.log4j2-conflict-handler :refer [log4j2-conflict-handler]])(:import- (java.io OutputStream)+ (java.io File OutputStream)(java.nio.file Files OpenOption StandardOpenOption)(java.util.jar Manifest)))@@ -47,7 +47,8 @@(defn- create-basis [edition]{:pre [(#{:ee :oss} edition)]}- (b/create-basis {:project "deps.edn", :aliases #{edition}}))+ (b/create-basis {:project "deps.edn",+ :aliases #{edition :drivers}}))(defn- all-paths [basis](concat (:paths basis)@@ -78,6 +79,22 @@(ns.deps/graph)ns-decls))+(def ^:private drivers-excluded-from-aot+ "Drivers whose JDBC dependencies are not bundled due to licensing restrictions (users must supply the JDBC driver JAR+ themselves). These drivers are included as source on the classpath and compiled lazily at runtime when their JDBC+ driver is present in the plugins directory."+ #{"oracle" "vertica"})++(defn- all-drivers []+ (->> (.listFiles (io/file (u/filename u/project-root-directory "modules" "drivers")))+ (filter (fn [^File d]+ (and+ (.isDirectory d)+ (not (.isHidden d))+ (.exists (io/file d "deps.edn"))+ (not (contains? drivers-excluded-from-aot (.getName d))))))+ (map (comp symbol #(str "metabase.driver." %) #(.getName ^File %)))))+(defn- metabase-namespaces-in-topo-order [basis](let [ns-decls (into [](comp (map io/file)@@ -88,7 +105,7 @@ns.deps/topo-sort(filter ns-symbols))orphans (remove (set sorted) ns-symbols)- all (concat orphans sorted)]+ all (concat orphans sorted (all-drivers))](assert (contains? (set all) 'metabase.core.bootstrap))(when (contains? ns-symbols 'metabase-enterprise.core.dummy-namespace)(assert (contains? (set all) 'metabase-enterprise.core.dummy-namespace)))@@ -121,7 +138,10 @@;; clean Docker env, so it's more of a nice to have to keep the clutter in our JARs down when building locally.#"\~$"#"^\.?#"- #"\.rej$"])+ #"\.rej$"+ ;; Driver classes are now flattened directly into the uberjar via the :drivers alias — the old nested+ ;; driver JARs in resources/modules/ must not be included or we'd ship everything twice.+ #"^modules/"])(defn- copy-resources! [basis](u/step "Copy resources"@@ -132,7 +152,14 @@(u/step (format "Copy resources from %s" path)(b/copy-dir {:target-dir class-dir:src-dirs [path]- :ignores resource-ignore-patterns})))))+ :ignores resource-ignore-patterns})))+ ;; Drivers excluded from AOT need their source files in the jar so they can be compiled lazily at runtime.+ (doseq [driver drivers-excluded-from-aot]+ (let [src-dir (u/filename u/project-root-directory "modules" "drivers" driver "src")]+ (when (.isDirectory (io/file src-dir))+ (u/step (format "Copy source for non-AOT driver %s" driver)+ (b/copy-dir {:target-dir class-dir+ :src-dirs [src-dir]})))))))(def ^:private dependency-ignore-patterns"Files to ignore when copying resources from our dependencies."@@ -141,7 +168,11 @@;; tf` -- see Slacc thread;; https://metaboat.slack.com/archives/C5XHN8GLW/p1731633690703149?thread_ts=1731504670.951389&cid=C5XHN8GLW#".*module-info\.class"- #"^LICENSE$"])+ #"^LICENSE$"+ #".*LICENSE.jol.txt"+ #"META-INF/license/LICENSE.jol.txt"+ #"META-INF/license.*"+ #"META-INF/LICENSE.*"])(defn- create-uberjar! [basis](u/step "Create uberjar"diff --git a/bin/build/test/build_drivers/build_driver_test.clj b/bin/build/test/build_drivers/build_driver_test.cljdeleted file mode 100644index 3dfc1317471..00000000000--- a/bin/build/test/build_drivers/build_driver_test.clj+++ /dev/null@@ -1,28 +0,0 @@-(ns build-drivers.build-driver-test- (:require- [build-drivers.build-driver :as build-driver]- [build-drivers.common :as c]- [build-drivers.verify :as verify]- [clojure.test :refer :all]))--(set! *warn-on-reflection* true)--(defn- jar-path ^String []- (c/driver-jar-destination-path :oracle))--(defn- jar-contains-jdbc-classes? []- (#'verify/jar-contains-file? (jar-path) "oracle/jdbc/OracleDriver.class"))--(deftest build-oss-driver-test- (testing "We should be able to build an OSS driver"- (build-driver/build-driver! :oracle :oss)- (is (.exists (java.io.File. (jar-path))))- (testing "JAR should not contain the JDBC driver classes"- (is (not (jar-contains-jdbc-classes?))))))--(deftest build-ee-driver-test- (testing "We should be able to build an EE driver"- (build-driver/build-driver! :oracle :ee)- (is (.exists (java.io.File. (jar-path))))- (testing "JAR *should* contain the JDBC driver classes"- (is (jar-contains-jdbc-classes?)))))diff --git a/deps.edn b/deps.ednindex 314b7ca1844..b69ae8b9529 100644--- a/deps.edn+++ b/deps.edn@@ -420,7 +420,11 @@;; include EE source code.:ee- {:extra-paths ["enterprise/backend/src"]}+ {:extra-paths ["enterprise/backend/src"]+ ;; make sure these stay in lockstep with their respective modules+ :extra-deps {com.oracle.database.jdbc/ojdbc17 {:mvn/version "23.9.0.25.07"}+ com.vertica.jdbc/vertica-jdbc+ ^:antq/exclude {:mvn/version "23.4.0-0"}}};; Include EE tests.;; for EE dev: `clojure -X:dev:ee:ee-dev`diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj b/enterprise/backend/src/metabase_enterprise/audit_app/audit.cljindex 0f0b4949453..1e637f14a68 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj@@ -23,19 +23,6 @@(set! *warn-on-reflection* true)-(defn- get-jar-path- "Returns the path to the currently running jar file.-- More info: https://stackoverflow.com/questions/320542/how-to-get-the-path-of-a-running-jar-file"- []- (assert (config/jar?) "Can only get-jar-path when running from a jar.")- (-> (class {})- (.getProtectionDomain)- (.getCodeSource)- (.getLocation)- (.toURI) ;; avoid problems with special characters in path.- (.getPath)))-(defn copy-from-jar!"Recursively copies a subdirectory (at resource-path) from the jar at jar-path into out-dir.@@ -199,8 +186,8 @@(let [ia-dir (instance-analytics-plugin-dir plugins-dir)](when (fs/exists? (u.files/relative-path ia-dir))(fs/delete-tree (u.files/relative-path ia-dir)))- (if (config/jar?)- (let [path-to-jar (get-jar-path)]+ (if (u.files/running-from-jar?)+ (let [path-to-jar (u.files/get-jar-path)](log/info "The app is running from a jar, starting copy...")(log/info (str "Copying " path-to-jar "::" jar-resource-path " -> " plugins-dir))(copy-from-jar! path-to-jar jar-resource-path plugins-dir)diff --git a/modules/drivers/athena/resources/metabase-plugin.yaml b/modules/drivers/athena/resources/metabase/athena/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/athena/resources/metabase-plugin.yamlrename to modules/drivers/athena/resources/metabase/athena/metabase-plugin.yamldiff --git a/modules/drivers/bigquery-cloud-sdk/resources/metabase-plugin.yaml b/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/bigquery-cloud-sdk/resources/metabase-plugin.yamlrename to modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yamldiff --git a/modules/drivers/clickhouse/resources/metabase-plugin.yaml b/modules/drivers/clickhouse/resources/metabase/clickhouse/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/clickhouse/resources/metabase-plugin.yamlrename to modules/drivers/clickhouse/resources/metabase/clickhouse/metabase-plugin.yamldiff --git a/modules/drivers/databricks/resources/metabase-plugin.yaml b/modules/drivers/databricks/resources/metabase/databricks/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/databricks/resources/metabase-plugin.yamlrename to modules/drivers/databricks/resources/metabase/databricks/metabase-plugin.yamldiff --git a/modules/drivers/druid-jdbc/resources/metabase-plugin.yaml b/modules/drivers/druid-jdbc/resources/metabase/druid-jdbc/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/druid-jdbc/resources/metabase-plugin.yamlrename to modules/drivers/druid-jdbc/resources/metabase/druid-jdbc/metabase-plugin.yamldiff --git a/modules/drivers/druid/resources/metabase-plugin.yaml b/modules/drivers/druid/resources/metabase/druid/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/druid/resources/metabase-plugin.yamlrename to modules/drivers/druid/resources/metabase/druid/metabase-plugin.yamldiff --git a/modules/drivers/hive-like/resources/metabase-plugin.yaml b/modules/drivers/hive-like/resources/metabase/hive-like/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/hive-like/resources/metabase-plugin.yamlrename to modules/drivers/hive-like/resources/metabase/hive-like/metabase-plugin.yamldiff --git a/modules/drivers/mongo/resources/metabase-plugin.yaml b/modules/drivers/mongo/resources/metabase/mongo/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/mongo/resources/metabase-plugin.yamlrename to modules/drivers/mongo/resources/metabase/mongo/metabase-plugin.yamldiff --git a/modules/drivers/oracle/deps.edn b/modules/drivers/oracle/deps.ednindex 071e6055dfe..ce3753b9ffe 100644--- a/modules/drivers/oracle/deps.edn+++ b/modules/drivers/oracle/deps.edn@@ -1,14 +1,13 @@{:paths- ["src" "resources-ee"]+ ["src" "resources"]:deps- {com.oracle.database.jdbc/ojdbc17 {:mvn/version "23.9.0.25.07"}}+ {}:aliases{:oss;; JDBC driver isn't GPL-compatible- {:replace-deps {}- :replace-paths ["src" "resources"]}+ {}:ee- {}}}+ {:extra-deps {com.oracle.database.jdbc/ojdbc17 {:mvn/version "23.9.0.25.07"}}}}}diff --git a/modules/drivers/oracle/resources-ee/metabase-plugin.yaml b/modules/drivers/oracle/resources-ee/metabase-plugin.yamldeleted file mode 100644index 289ce6e856a..00000000000--- a/modules/drivers/oracle/resources-ee/metabase-plugin.yaml+++ /dev/null@@ -1,74 +0,0 @@-info:- name: Metabase Oracle Driver- version: 1.0.0-SNAPSHOT- description: Allows Metabase to connect to Oracle databases.-driver:- name: oracle- display-name: Oracle- lazy-load: true- parent: sql-jdbc- connection-properties:- - host- - merge:- - port- - default: 1521- - name: sid- display-name: Oracle system ID (SID)- placeholder: Usually something like ORCL or XE. Optional if using service name- - name: service-name- display-name: Oracle service name- placeholder: Optional TNS alias- - merge:- - user- - required: false- - merge:- - password- - required: false- - cloud-ip-address-info- - ssl- - name: ssl-use-keystore- display-name: Use SSL server certificate?- type: boolean- visible-if:- ssl: true- - name: ssl-keystore- display-name: Keystore (PKCS12 or JKS)- type: secret- secret-kind: keystore- placeholder: /path/to/keystore- visible-if:- ssl-use-keystore: true- - name: ssl-keystore-password- display-name: Keystore password- type: secret- secret-kind: password- required: false- visible-if:- ssl-use-keystore: true- - name: ssl-use-truststore- display-name: Use SSL truststore?- type: boolean- visible-if:- ssl: true- - name: ssl-truststore- display-name: Truststore (PKCS12 or JKS)- type: secret- secret-kind: keystore- placeholder: /path/to/truststore- visible-if:- ssl-use-truststore: true- - name: ssl-truststore-password- display-name: Truststore password- type: secret- secret-kind: password- required: false- visible-if:- ssl-use-truststore: true- - ssh-tunnel- - advanced-options-start- - default-advanced-options-init:- - step: load-namespace- namespace: metabase.driver.oracle- - step: register-jdbc-driver- class: oracle.jdbc.OracleDriverdiff --git a/modules/drivers/oracle/resources/metabase-plugin.yaml b/modules/drivers/oracle/resources/metabase/oracle/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/oracle/resources/metabase-plugin.yamlrename to modules/drivers/oracle/resources/metabase/oracle/metabase-plugin.yamldiff --git a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj b/modules/drivers/oracle/test/metabase/driver/oracle_test.cljindex 1789f55682c..b282c7c1deb 100644--- a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj+++ b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj@@ -111,8 +111,9 @@(deftest connection-properties-test(testing "Connection properties should be returned properly (including transformation of secret types)"(with-redefs [premium-features/is-hosted? (constantly false)]- (let [expected [{:name "host"}- {:name "port"}+ (let [expected [{:type :group+ :fields [{:name "host"}+ {:name "port"}]}{:name "sid"}{:name "service-name"}{:name "user"}@@ -185,7 +186,7 @@(driver.u/connection-props-server->client :oracle))](is (= (count expected) (count actual))(str "actual names: " (pr-str (mapv :name actual))))- (is (= expected (mt/select-keys-sequentially expected actual)))))))+ (is (=? expected actual))))))(deftest ^:parallel test-ssh-connection(testing "Gets an error when it can't connect to oracle via ssh tunnel"diff --git a/modules/drivers/presto-jdbc/resources/metabase-plugin.yaml b/modules/drivers/presto-jdbc/resources/metabase/presto-jdbc/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/presto-jdbc/resources/metabase-plugin.yamlrename to modules/drivers/presto-jdbc/resources/metabase/presto-jdbc/metabase-plugin.yamldiff --git a/modules/drivers/redshift/resources/metabase-plugin.yaml b/modules/drivers/redshift/resources/metabase/redshift/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/redshift/resources/metabase-plugin.yamlrename to modules/drivers/redshift/resources/metabase/redshift/metabase-plugin.yamldiff --git a/modules/drivers/snowflake/resources/metabase-plugin.yaml b/modules/drivers/snowflake/resources/metabase/snowflake/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/snowflake/resources/metabase-plugin.yamlrename to modules/drivers/snowflake/resources/metabase/snowflake/metabase-plugin.yamldiff --git a/modules/drivers/sparksql/resources/metabase-plugin.yaml b/modules/drivers/sparksql/resources/metabase/sparksql/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/sparksql/resources/metabase-plugin.yamlrename to modules/drivers/sparksql/resources/metabase/sparksql/metabase-plugin.yamldiff --git a/modules/drivers/sqlite/resources/metabase-plugin.yaml b/modules/drivers/sqlite/resources/metabase/sqlite/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/sqlite/resources/metabase-plugin.yamlrename to modules/drivers/sqlite/resources/metabase/sqlite/metabase-plugin.yamldiff --git a/modules/drivers/sqlserver/resources/metabase-plugin.yaml b/modules/drivers/sqlserver/resources/metabase/sqlserver/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/sqlserver/resources/metabase-plugin.yamlrename to modules/drivers/sqlserver/resources/metabase/sqlserver/metabase-plugin.yamldiff --git a/modules/drivers/starburst/resources/metabase-plugin.yaml b/modules/drivers/starburst/resources/metabase/starburst/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/starburst/resources/metabase-plugin.yamlrename to modules/drivers/starburst/resources/metabase/starburst/metabase-plugin.yamldiff --git a/modules/drivers/vertica/deps.edn b/modules/drivers/vertica/deps.ednindex fcf1690b01c..a80ed2e496a 100644--- a/modules/drivers/vertica/deps.edn+++ b/modules/drivers/vertica/deps.edn@@ -1,15 +1,14 @@{:paths- ["src" "resources-ee"]+ ["src" "resources"]:deps- {com.vertica.jdbc/vertica-jdbc ^:antq/exclude ; update to 24.3 breaks the date tests- {:mvn/version "23.4.0-0"}}+ {}:aliases{:oss;; JDBC driver isn't GPL-compatible- {:replace-deps {}- :replace-paths ["src" "resources"]}+ {}:ee- {}}}+ {:extra-deps {com.vertica.jdbc/vertica-jdbc ^:antq/exclude ; update to 24.3 breaks the date tests+ {:mvn/version "23.4.0-0"}}}}}diff --git a/modules/drivers/vertica/resources-ee/metabase-plugin.yaml b/modules/drivers/vertica/resources-ee/metabase-plugin.yamldeleted file mode 100644index 8ad3ba627fe..00000000000--- a/modules/drivers/vertica/resources-ee/metabase-plugin.yaml+++ /dev/null@@ -1,29 +0,0 @@-info:- name: Metabase Vertica Driver- version: 1.0.0-SNAPSHOT- description: Allows Metabase to connect to Vertica databases.-driver:- name: vertica- display-name: Vertica- lazy-load: true- parent: sql-jdbc- connection-properties:- - host- - merge:- - port- - default: 5433- - dbname- - user- - password- - cloud-ip-address-info- - ssh-tunnel- - advanced-options-start- - merge:- - additional-options- - placeholder: 'ConnectionLoadBalance=1'- - default-advanced-options-init:- - step: load-namespace- namespace: metabase.driver.vertica- - step: register-jdbc-driver- class: com.vertica.jdbc.Driverdiff --git a/modules/drivers/vertica/resources/metabase-plugin.yaml b/modules/drivers/vertica/resources/metabase/vertica/metabase-plugin.yamlsimilarity index 100%rename from modules/drivers/vertica/resources/metabase-plugin.yamlrename to modules/drivers/vertica/resources/metabase/vertica/metabase-plugin.yamldiff --git a/src/metabase/plugins/impl.clj b/src/metabase/plugins/impl.cljindex 9806764f521..e006ed8fef9 100644--- a/src/metabase/plugins/impl.clj+++ b/src/metabase/plugins/impl.clj@@ -6,7 +6,6 @@[clojure.string :as str][environ.core :as env][metabase.classloader.core :as classloader]- [metabase.config.core :as config][metabase.plugins.initialize :as plugins.init][metabase.util.files :as u.files][metabase.util.i18n :refer [trs]]@@ -14,7 +13,7 @@[metabase.util.yaml :as yaml])(:import(java.io File)- (java.nio.file Files Path)))+ (java.nio.file Files FileVisitOption Path Paths)))(set! *warn-on-reflection* true)@@ -65,12 +64,6 @@[](:path (plugins-dir-info)))-(defn- extract-system-modules! []- (when (io/resource "modules")- (let [plugins-path (plugins-dir)]- (u.files/with-open-path-to-resource [modules-path "modules"]- (u.files/copy-files! modules-path plugins-path)))))-;;; +----------------------------------------------------------------------------------------------------------------+;;; | loading/initializing plugins |;;; +----------------------------------------------------------------------------------------------------------------+@@ -115,40 +108,36 @@"spark-deps.jar is no longer needed by Metabase 0.32.0+. You can delete it from the plugins directory.")))]path))-(when (not config/is-prod?)- (defn- load-local-plugin-manifest! [^Path path]- (some-> (slurp (str path)) yaml/parse-string plugins.init/init-plugin-with-info!))-- (defn- driver-manifest-paths- "Return a sequence of [[java.io.File]] paths for `metabase-plugin.yaml` plugin manifests for drivers on the classpath."- []- ;; only include plugin manifests if they're on the system classpath.- (concat- (for [^File file (classpath/system-classpath)- :when (and (.isDirectory file)- (not (.isHidden file))- (str/includes? (str file) "modules/drivers")- (or (str/ends-with? (str file) "resources")- (str/ends-with? (str file) "resources-ee")))- :let [manifest-file (io/file file "metabase-plugin.yaml")]- :when (.exists manifest-file)]- manifest-file)- ;; for hacking on 3rd-party drivers locally: set- ;; `-Dmb.dev.additional.driver.manifest.paths=/path/to/whatever/metabase-plugin.yaml` or- ;; `MB_DEV_ADDITIONAL_DRIVER_MANIFEST_PATHS=...` to have that plugin manifest get loaded during startup. Specify- ;; multiple plugin manifests by comma-separating them.- (when-let [additional-paths (env/env :mb-dev-additional-driver-manifest-paths)]- (map u.files/get-path (str/split additional-paths #",")))))-- (defn- load-local-plugin-manifests!- "Load local plugin manifest files when not running in a production mode, to simulate what would happen when loading those- same plugins from the uberjar. This is needed because some plugin manifests define driver methods and the like that- aren't defined elsewhere."- []- ;; TODO - this should probably do an actual search in case we ever add any additional directories- (doseq [manifest-path (driver-manifest-paths)]- (log/infof "Loading local plugin manifest at %s" (str manifest-path))- (load-local-plugin-manifest! manifest-path))))+(defn- load-plugin-manifest! [path]+ (some-> (slurp (str path)) yaml/parse-string plugins.init/init-plugin-with-info!))++(defn- bundled-driver-manifest-paths+ "Return a sequence of paths (URIs when running from a jar, Paths in dev) for `metabase-plugin.yaml` plugin manifests+ for drivers on the classpath. Driver manifests live at `metabase/<driver-name>/metabase-plugin.yaml` on the classpath."+ []+ (if (u.files/running-from-jar?)+ (u.files/find-in-current-jar "glob:/metabase/*/metabase-plugin.yaml")+ (let [matcher (.getPathMatcher (java.nio.file.FileSystems/getDefault) "glob:**/metabase/*/metabase-plugin.yaml")]+ (for [^File f (classpath/system-classpath)+ :when (.isDirectory f)+ founds [(-> (.toURI f)+ (Paths/get)+ (Files/find 3+ (fn [path _attr] (.matches matcher path))+ (make-array FileVisitOption 0))+ (.collect (java.util.stream.Collectors/toList)))]+ f founds]+ f))))++(defn- load-bundled-driver-plugin-manifests!+ "Find and load driver plugin manifests from the classpath. In the uberjar, driver classes are flattened directly into+ the jar (rather than shipped as nested JARs in a `modules/` directory) and their manifests live at+ `metabase/<driver>/metabase-plugin.yaml`. In dev, the same manifests are found via the driver resource directories on+ the classpath."+ []+ (doseq [manifest-path (bundled-driver-manifest-paths)]+ (log/infof "Loading driver plugin manifest at %s" (str manifest-path))+ (load-plugin-manifest! manifest-path)))(defn- has-manifest? ^Boolean [^Path path](boolean (u.files/file-exists-in-archive? path "metabase-plugin.yaml")))@@ -166,29 +155,29 @@(log/errorf e "Failed to initialize plugin %s" (.getFileName path))))))(defn- load! []+ ;; Load any user-supplied plugin JARs from the plugins directory (e.g. Oracle JDBC driver).+ ;; System/bundled drivers are no longer extracted to disk — they are flattened into the uberjar+ ;; and their manifests are loaded from the classpath via load-driver-plugin-manifests! below.(log/infof "Loading plugins in %s..." (str (plugins-dir)))- (extract-system-modules!)(let [paths (plugins-paths)](init-plugins! paths))- (when (not config/is-prod?)- (load-local-plugin-manifests!)))+ (load-bundled-driver-plugin-manifests!))(defonce ^:private loaded? (atom false))(defn load-plugins!- "Load Metabase plugins. The are JARs shipped as part of Metabase itself, under the `resources/modules` directory (the- source for these JARs is under the `modules` directory); and others manually added by users to the Metabase plugins- directory, which defaults to `./plugins`.+ "Load Metabase plugins. Bundled drivers are compiled directly into the uberjar and their plugin manifests are+ discovered on the classpath at `metabase/<driver>/metabase-plugin.yaml`. User-supplied plugins (e.g. Oracle JDBC+ driver JARs) are loaded from the plugins directory, which defaults to `./plugins`.When loading plugins, Metabase performs the following steps:* Metabase creates the plugins directory if it does not already exist.- * Any plugins that are shipped as part of Metabase itself are extracted from the Metabase uberjar (or `resources`- directory when running with the Clojure CLI) into the plugins directory.* Each JAR in the plugins directory that *does not* include a Metabase plugin manifest is added to the classpath.* For JARs that include a Metabase plugin manifest (a `metabase-plugin.yaml` file), a lazy-loading Metabase driveris registered; when the driver is initialized (automatically, when certain methods are called) the JAR is added- to the classpath and the driver namespace is loaded+ to the classpath and the driver namespace is loaded.+ * Bundled driver plugin manifests are loaded from the classpath (not from disk).This function will only perform loading steps the first time it is called — it is safe to call this function morethan once."diff --git a/src/metabase/util/files.clj b/src/metabase/util/files.cljindex 0c41718ae12..519ee72516c 100644--- a/src/metabase/util/files.clj+++ b/src/metabase/util/files.clj@@ -15,8 +15,8 @@(:import(java.io FileNotFoundException)(java.net URL)- (java.nio.file CopyOption Files FileSystem FileSystemAlreadyExistsException FileSystems- LinkOption OpenOption Path Paths StandardCopyOption)+ (java.nio.file CopyOption Files FileSystem FileSystemAlreadyExistsException+ FileSystems LinkOption OpenOption Path Paths StandardCopyOption)(java.nio.file.attribute FileAttribute)(java.util Collections)(java.util.zip ZipInputStream)))@@ -183,3 +183,55 @@"Returns a java.nio.file.Path"[path](fs/relativize (fs/absolutize ".") path))++(defn running-from-jar?+ "Returns true iff we are running from a jar.++ .getResource will return a java.net.URL, and those start with \"jar:\" if and only if the app is running from a jar.++ More info: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/Thread.html"+ []+ (= "jar" (.. (Thread/currentThread)+ getContextClassLoader+ (getResource ".keep-me")+ getProtocol)))++(defn get-jar-path+ "Returns the path to the currently running jar file.++ More info: https://stackoverflow.com/questions/320542/how-to-get-the-path-of-a-running-jar-file"+ []+ (assert (running-from-jar?) "Can only get-jar-path when running from a jar.")+ (-> (class {})+ (.getProtectionDomain)+ (.getCodeSource)+ (.getLocation)+ (.toURI) ;; avoid problems with special characters in path.+ (.getPath)))++(defn find-in-current-jar+ "Find matching files in the current jar. See+ https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String- to+ understand the syntax of [[pattern]]. Motivating use case is:++ `(u.files/find-in-current-jar \"glob:/metabase/*/metabase-plugin.yaml\")`++ to find the plugin manifests in the jar."+ [pattern]+ (let [jar-url (URL. (str "jar:file:" (get-jar-path) "!/"))]+ (with-open [fs (jar-file-system-from-url jar-url)]+ (let [matcher (.getPathMatcher fs pattern)+ root (.getPath fs "/" (into-array String []))+ files (atom [])]+ (Files/walkFileTree root+ (proxy [java.nio.file.SimpleFileVisitor] []+ (preVisitDirectory [dir _attrs]+ (if (or (str/starts-with? (str dir) "/metabase")+ (= (str "/") (str dir)))+ java.nio.file.FileVisitResult/CONTINUE+ java.nio.file.FileVisitResult/SKIP_SUBTREE))+ (visitFile [file _attrs]+ (when (.matches matcher file)+ (swap! files conj file))+ java.nio.file.FileVisitResult/CONTINUE)))+ (into [] (map #(.toUri ^Path %) @files))))))