Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Drivers on classpath (#73519)

dpsutton823c6f69ae0a
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 5da604f5647..4f4bb54cbe2 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1516,7 +1516,6 @@
{:team "DevEx"
:api #{metabase.plugins.core}
:uses #{classloader
- config
driver
util}}
diff --git a/bin/build/src/build.clj b/bin/build/src/build.clj
index a22c709f0a3..e71bae38036 100644
--- a/bin/build/src/build.clj
+++ b/bin/build/src/build.clj
@@ -1,6 +1,5 @@
(ns build
(:require
- [build-drivers :as build-drivers]
[build.licenses :as license]
[build.python :as build.python]
[build.uberjar :as uberjar]
@@ -89,8 +88,8 @@
(build-frontend! edition))
:licenses (fn [{:keys [edition]}]
(build-licenses! edition))
- :drivers (fn [{:keys [edition]}]
- (build-drivers/build-drivers! edition))
+ #_#_:drivers (fn [{:keys [edition]}]
+ (build-drivers/build-drivers! edition))
:python (fn [{:keys [edition]}]
(build.python/build-python-deps! edition))
:uberjar (fn [{:keys [edition]}]
diff --git a/bin/build/src/build/uberjar.clj b/bin/build/src/build/uberjar.clj
index 0d7f2fe0a5b..b87d653ab3a 100644
--- a/bin/build/src/build/uberjar.clj
+++ b/bin/build/src/build/uberjar.clj
@@ -9,7 +9,7 @@
[metabuild-common.core :as u]
[org.corfield.log4j2-conflict-handler :refer [log4j2-conflict-handler]])
(:import
- (java.io OutputStream)
+ (java.io File OutputStream)
(java.nio.file Files OpenOption StandardOpenOption)
(java.util.jar Manifest)))
@@ -47,7 +47,8 @@
(defn- create-basis [edition]
{:pre [(#{:ee :oss} edition)]}
- (b/create-basis {:project "deps.edn", :aliases #{edition}}))
+ (b/create-basis {:project "deps.edn",
+ :aliases #{edition :drivers}}))
(defn- all-paths [basis]
(concat (:paths basis)
@@ -78,6 +79,22 @@
(ns.deps/graph)
ns-decls))
+(def ^:private drivers-excluded-from-aot
+ "Drivers whose JDBC dependencies are not bundled due to licensing restrictions (users must supply the JDBC driver JAR
+ themselves). These drivers are included as source on the classpath and compiled lazily at runtime when their JDBC
+ driver is present in the plugins directory."
+ #{"oracle" "vertica"})
+
+(defn- all-drivers []
+ (->> (.listFiles (io/file (u/filename u/project-root-directory "modules" "drivers")))
+ (filter (fn [^File d]
+ (and
+ (.isDirectory d)
+ (not (.isHidden d))
+ (.exists (io/file d "deps.edn"))
+ (not (contains? drivers-excluded-from-aot (.getName d))))))
+ (map (comp symbol #(str "metabase.driver." %) #(.getName ^File %)))))
+
(defn- metabase-namespaces-in-topo-order [basis]
(let [ns-decls (into []
(comp (map io/file)
@@ -88,7 +105,7 @@
ns.deps/topo-sort
(filter ns-symbols))
orphans (remove (set sorted) ns-symbols)
- all (concat orphans sorted)]
+ all (concat orphans sorted (all-drivers))]
(assert (contains? (set all) 'metabase.core.bootstrap))
(when (contains? ns-symbols 'metabase-enterprise.core.dummy-namespace)
(assert (contains? (set all) 'metabase-enterprise.core.dummy-namespace)))
@@ -121,7 +138,10 @@
;; clean Docker env, so it's more of a nice to have to keep the clutter in our JARs down when building locally.
#"\~$"
#"^\.?#"
- #"\.rej$"])
+ #"\.rej$"
+ ;; Driver classes are now flattened directly into the uberjar via the :drivers alias — the old nested
+ ;; driver JARs in resources/modules/ must not be included or we'd ship everything twice.
+ #"^modules/"])
(defn- copy-resources! [basis]
(u/step "Copy resources"
@@ -132,7 +152,14 @@
(u/step (format "Copy resources from %s" path)
(b/copy-dir {:target-dir class-dir
:src-dirs [path]
- :ignores resource-ignore-patterns})))))
+ :ignores resource-ignore-patterns})))
+ ;; Drivers excluded from AOT need their source files in the jar so they can be compiled lazily at runtime.
+ (doseq [driver drivers-excluded-from-aot]
+ (let [src-dir (u/filename u/project-root-directory "modules" "drivers" driver "src")]
+ (when (.isDirectory (io/file src-dir))
+ (u/step (format "Copy source for non-AOT driver %s" driver)
+ (b/copy-dir {:target-dir class-dir
+ :src-dirs [src-dir]})))))))
(def ^:private dependency-ignore-patterns
"Files to ignore when copying resources from our dependencies."
@@ -141,7 +168,11 @@
;; tf` -- see Slacc thread
;; https://metaboat.slack.com/archives/C5XHN8GLW/p1731633690703149?thread_ts=1731504670.951389&cid=C5XHN8GLW
#".*module-info\.class"
- #"^LICENSE$"])
+ #"^LICENSE$"
+ #".*LICENSE.jol.txt"
+ #"META-INF/license/LICENSE.jol.txt"
+ #"META-INF/license.*"
+ #"META-INF/LICENSE.*"])
(defn- create-uberjar! [basis]
(u/step "Create uberjar"
diff --git a/bin/build/test/build_drivers/build_driver_test.clj b/bin/build/test/build_drivers/build_driver_test.clj
deleted file mode 100644
index 3dfc1317471..00000000000
--- a/bin/build/test/build_drivers/build_driver_test.clj
+++ /dev/null
@@ -1,28 +0,0 @@
-(ns build-drivers.build-driver-test
- (:require
- [build-drivers.build-driver :as build-driver]
- [build-drivers.common :as c]
- [build-drivers.verify :as verify]
- [clojure.test :refer :all]))
-
-(set! *warn-on-reflection* true)
-
-(defn- jar-path ^String []
- (c/driver-jar-destination-path :oracle))
-
-(defn- jar-contains-jdbc-classes? []
- (#'verify/jar-contains-file? (jar-path) "oracle/jdbc/OracleDriver.class"))
-
-(deftest build-oss-driver-test
- (testing "We should be able to build an OSS driver"
- (build-driver/build-driver! :oracle :oss)
- (is (.exists (java.io.File. (jar-path))))
- (testing "JAR should not contain the JDBC driver classes"
- (is (not (jar-contains-jdbc-classes?))))))
-
-(deftest build-ee-driver-test
- (testing "We should be able to build an EE driver"
- (build-driver/build-driver! :oracle :ee)
- (is (.exists (java.io.File. (jar-path))))
- (testing "JAR *should* contain the JDBC driver classes"
- (is (jar-contains-jdbc-classes?)))))
diff --git a/deps.edn b/deps.edn
index 314b7ca1844..b69ae8b9529 100644
--- a/deps.edn
+++ b/deps.edn
@@ -420,7 +420,11 @@
;; include EE source code.
:ee
- {:extra-paths ["enterprise/backend/src"]}
+ {:extra-paths ["enterprise/backend/src"]
+ ;; make sure these stay in lockstep with their respective modules
+ :extra-deps {com.oracle.database.jdbc/ojdbc17 {:mvn/version "23.9.0.25.07"}
+ com.vertica.jdbc/vertica-jdbc
+ ^:antq/exclude {:mvn/version "23.4.0-0"}}}
;; Include EE tests.
;; for EE dev: `clojure -X:dev:ee:ee-dev`
diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj b/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj
index 0f0b4949453..1e637f14a68 100644
--- a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj
+++ b/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj
@@ -23,19 +23,6 @@
(set! *warn-on-reflection* true)
-(defn- get-jar-path
- "Returns the path to the currently running jar file.
-
- More info: https://stackoverflow.com/questions/320542/how-to-get-the-path-of-a-running-jar-file"
- []
- (assert (config/jar?) "Can only get-jar-path when running from a jar.")
- (-> (class {})
- (.getProtectionDomain)
- (.getCodeSource)
- (.getLocation)
- (.toURI) ;; avoid problems with special characters in path.
- (.getPath)))
-
(defn copy-from-jar!
"Recursively copies a subdirectory (at resource-path) from the jar at jar-path into out-dir.
@@ -199,8 +186,8 @@
(let [ia-dir (instance-analytics-plugin-dir plugins-dir)]
(when (fs/exists? (u.files/relative-path ia-dir))
(fs/delete-tree (u.files/relative-path ia-dir)))
- (if (config/jar?)
- (let [path-to-jar (get-jar-path)]
+ (if (u.files/running-from-jar?)
+ (let [path-to-jar (u.files/get-jar-path)]
(log/info "The app is running from a jar, starting copy...")
(log/info (str "Copying " path-to-jar "::" jar-resource-path " -> " plugins-dir))
(copy-from-jar! path-to-jar jar-resource-path plugins-dir)
diff --git a/modules/drivers/athena/resources/metabase-plugin.yaml b/modules/drivers/athena/resources/metabase/athena/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/athena/resources/metabase-plugin.yaml
rename to modules/drivers/athena/resources/metabase/athena/metabase-plugin.yaml
diff --git a/modules/drivers/bigquery-cloud-sdk/resources/metabase-plugin.yaml b/modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/bigquery-cloud-sdk/resources/metabase-plugin.yaml
rename to modules/drivers/bigquery-cloud-sdk/resources/metabase/bigquery-cloud-sdk/metabase-plugin.yaml
diff --git a/modules/drivers/clickhouse/resources/metabase-plugin.yaml b/modules/drivers/clickhouse/resources/metabase/clickhouse/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/clickhouse/resources/metabase-plugin.yaml
rename to modules/drivers/clickhouse/resources/metabase/clickhouse/metabase-plugin.yaml
diff --git a/modules/drivers/databricks/resources/metabase-plugin.yaml b/modules/drivers/databricks/resources/metabase/databricks/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/databricks/resources/metabase-plugin.yaml
rename to modules/drivers/databricks/resources/metabase/databricks/metabase-plugin.yaml
diff --git a/modules/drivers/druid-jdbc/resources/metabase-plugin.yaml b/modules/drivers/druid-jdbc/resources/metabase/druid-jdbc/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/druid-jdbc/resources/metabase-plugin.yaml
rename to modules/drivers/druid-jdbc/resources/metabase/druid-jdbc/metabase-plugin.yaml
diff --git a/modules/drivers/druid/resources/metabase-plugin.yaml b/modules/drivers/druid/resources/metabase/druid/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/druid/resources/metabase-plugin.yaml
rename to modules/drivers/druid/resources/metabase/druid/metabase-plugin.yaml
diff --git a/modules/drivers/hive-like/resources/metabase-plugin.yaml b/modules/drivers/hive-like/resources/metabase/hive-like/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/hive-like/resources/metabase-plugin.yaml
rename to modules/drivers/hive-like/resources/metabase/hive-like/metabase-plugin.yaml
diff --git a/modules/drivers/mongo/resources/metabase-plugin.yaml b/modules/drivers/mongo/resources/metabase/mongo/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/mongo/resources/metabase-plugin.yaml
rename to modules/drivers/mongo/resources/metabase/mongo/metabase-plugin.yaml
diff --git a/modules/drivers/oracle/deps.edn b/modules/drivers/oracle/deps.edn
index 071e6055dfe..ce3753b9ffe 100644
--- a/modules/drivers/oracle/deps.edn
+++ b/modules/drivers/oracle/deps.edn
@@ -1,14 +1,13 @@
{:paths
- ["src" "resources-ee"]
+ ["src" "resources"]
:deps
- {com.oracle.database.jdbc/ojdbc17 {:mvn/version "23.9.0.25.07"}}
+ {}
:aliases
{:oss
;; JDBC driver isn't GPL-compatible
- {:replace-deps {}
- :replace-paths ["src" "resources"]}
+ {}
:ee
- {}}}
+ {:extra-deps {com.oracle.database.jdbc/ojdbc17 {:mvn/version "23.9.0.25.07"}}}}}
diff --git a/modules/drivers/oracle/resources-ee/metabase-plugin.yaml b/modules/drivers/oracle/resources-ee/metabase-plugin.yaml
deleted file mode 100644
index 289ce6e856a..00000000000
--- a/modules/drivers/oracle/resources-ee/metabase-plugin.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-info:
- name: Metabase Oracle Driver
- version: 1.0.0-SNAPSHOT
- description: Allows Metabase to connect to Oracle databases.
-driver:
- name: oracle
- display-name: Oracle
- lazy-load: true
- parent: sql-jdbc
- connection-properties:
- - host
- - merge:
- - port
- - default: 1521
- - name: sid
- display-name: Oracle system ID (SID)
- placeholder: Usually something like ORCL or XE. Optional if using service name
- - name: service-name
- display-name: Oracle service name
- placeholder: Optional TNS alias
- - merge:
- - user
- - required: false
- - merge:
- - password
- - required: false
- - cloud-ip-address-info
- - ssl
- - name: ssl-use-keystore
- display-name: Use SSL server certificate?
- type: boolean
- visible-if:
- ssl: true
- - name: ssl-keystore
- display-name: Keystore (PKCS12 or JKS)
- type: secret
- secret-kind: keystore
- placeholder: /path/to/keystore
- visible-if:
- ssl-use-keystore: true
- - name: ssl-keystore-password
- display-name: Keystore password
- type: secret
- secret-kind: password
- required: false
- visible-if:
- ssl-use-keystore: true
- - name: ssl-use-truststore
- display-name: Use SSL truststore?
- type: boolean
- visible-if:
- ssl: true
- - name: ssl-truststore
- display-name: Truststore (PKCS12 or JKS)
- type: secret
- secret-kind: keystore
- placeholder: /path/to/truststore
- visible-if:
- ssl-use-truststore: true
- - name: ssl-truststore-password
- display-name: Truststore password
- type: secret
- secret-kind: password
- required: false
- visible-if:
- ssl-use-truststore: true
- - ssh-tunnel
- - advanced-options-start
- - default-advanced-options
-init:
- - step: load-namespace
- namespace: metabase.driver.oracle
- - step: register-jdbc-driver
- class: oracle.jdbc.OracleDriver
diff --git a/modules/drivers/oracle/resources/metabase-plugin.yaml b/modules/drivers/oracle/resources/metabase/oracle/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/oracle/resources/metabase-plugin.yaml
rename to modules/drivers/oracle/resources/metabase/oracle/metabase-plugin.yaml
diff --git a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj
index 1789f55682c..b282c7c1deb 100644
--- a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj
+++ b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj
@@ -111,8 +111,9 @@
(deftest connection-properties-test
(testing "Connection properties should be returned properly (including transformation of secret types)"
(with-redefs [premium-features/is-hosted? (constantly false)]
- (let [expected [{:name "host"}
- {:name "port"}
+ (let [expected [{:type :group
+ :fields [{:name "host"}
+ {:name "port"}]}
{:name "sid"}
{:name "service-name"}
{:name "user"}
@@ -185,7 +186,7 @@
(driver.u/connection-props-server->client :oracle))]
(is (= (count expected) (count actual))
(str "actual names: " (pr-str (mapv :name actual))))
- (is (= expected (mt/select-keys-sequentially expected actual)))))))
+ (is (=? expected actual))))))
(deftest ^:parallel test-ssh-connection
(testing "Gets an error when it can't connect to oracle via ssh tunnel"
diff --git a/modules/drivers/presto-jdbc/resources/metabase-plugin.yaml b/modules/drivers/presto-jdbc/resources/metabase/presto-jdbc/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/presto-jdbc/resources/metabase-plugin.yaml
rename to modules/drivers/presto-jdbc/resources/metabase/presto-jdbc/metabase-plugin.yaml
diff --git a/modules/drivers/redshift/resources/metabase-plugin.yaml b/modules/drivers/redshift/resources/metabase/redshift/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/redshift/resources/metabase-plugin.yaml
rename to modules/drivers/redshift/resources/metabase/redshift/metabase-plugin.yaml
diff --git a/modules/drivers/snowflake/resources/metabase-plugin.yaml b/modules/drivers/snowflake/resources/metabase/snowflake/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/snowflake/resources/metabase-plugin.yaml
rename to modules/drivers/snowflake/resources/metabase/snowflake/metabase-plugin.yaml
diff --git a/modules/drivers/sparksql/resources/metabase-plugin.yaml b/modules/drivers/sparksql/resources/metabase/sparksql/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/sparksql/resources/metabase-plugin.yaml
rename to modules/drivers/sparksql/resources/metabase/sparksql/metabase-plugin.yaml
diff --git a/modules/drivers/sqlite/resources/metabase-plugin.yaml b/modules/drivers/sqlite/resources/metabase/sqlite/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/sqlite/resources/metabase-plugin.yaml
rename to modules/drivers/sqlite/resources/metabase/sqlite/metabase-plugin.yaml
diff --git a/modules/drivers/sqlserver/resources/metabase-plugin.yaml b/modules/drivers/sqlserver/resources/metabase/sqlserver/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/sqlserver/resources/metabase-plugin.yaml
rename to modules/drivers/sqlserver/resources/metabase/sqlserver/metabase-plugin.yaml
diff --git a/modules/drivers/starburst/resources/metabase-plugin.yaml b/modules/drivers/starburst/resources/metabase/starburst/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/starburst/resources/metabase-plugin.yaml
rename to modules/drivers/starburst/resources/metabase/starburst/metabase-plugin.yaml
diff --git a/modules/drivers/vertica/deps.edn b/modules/drivers/vertica/deps.edn
index fcf1690b01c..a80ed2e496a 100644
--- a/modules/drivers/vertica/deps.edn
+++ b/modules/drivers/vertica/deps.edn
@@ -1,15 +1,14 @@
{:paths
- ["src" "resources-ee"]
+ ["src" "resources"]
:deps
- {com.vertica.jdbc/vertica-jdbc ^:antq/exclude ; update to 24.3 breaks the date tests
- {:mvn/version "23.4.0-0"}}
+ {}
:aliases
{:oss
;; JDBC driver isn't GPL-compatible
- {:replace-deps {}
- :replace-paths ["src" "resources"]}
+ {}
:ee
- {}}}
+ {:extra-deps {com.vertica.jdbc/vertica-jdbc ^:antq/exclude ; update to 24.3 breaks the date tests
+ {:mvn/version "23.4.0-0"}}}}}
diff --git a/modules/drivers/vertica/resources-ee/metabase-plugin.yaml b/modules/drivers/vertica/resources-ee/metabase-plugin.yaml
deleted file mode 100644
index 8ad3ba627fe..00000000000
--- a/modules/drivers/vertica/resources-ee/metabase-plugin.yaml
+++ /dev/null
@@ -1,29 +0,0 @@
-info:
- name: Metabase Vertica Driver
- version: 1.0.0-SNAPSHOT
- description: Allows Metabase to connect to Vertica databases.
-driver:
- name: vertica
- display-name: Vertica
- lazy-load: true
- parent: sql-jdbc
- connection-properties:
- - host
- - merge:
- - port
- - default: 5433
- - dbname
- - user
- - password
- - cloud-ip-address-info
- - ssh-tunnel
- - advanced-options-start
- - merge:
- - additional-options
- - placeholder: 'ConnectionLoadBalance=1'
- - default-advanced-options
-init:
- - step: load-namespace
- namespace: metabase.driver.vertica
- - step: register-jdbc-driver
- class: com.vertica.jdbc.Driver
diff --git a/modules/drivers/vertica/resources/metabase-plugin.yaml b/modules/drivers/vertica/resources/metabase/vertica/metabase-plugin.yaml
similarity index 100%
rename from modules/drivers/vertica/resources/metabase-plugin.yaml
rename to modules/drivers/vertica/resources/metabase/vertica/metabase-plugin.yaml
diff --git a/src/metabase/plugins/impl.clj b/src/metabase/plugins/impl.clj
index 9806764f521..e006ed8fef9 100644
--- a/src/metabase/plugins/impl.clj
+++ b/src/metabase/plugins/impl.clj
@@ -6,7 +6,6 @@
[clojure.string :as str]
[environ.core :as env]
[metabase.classloader.core :as classloader]
- [metabase.config.core :as config]
[metabase.plugins.initialize :as plugins.init]
[metabase.util.files :as u.files]
[metabase.util.i18n :refer [trs]]
@@ -14,7 +13,7 @@
[metabase.util.yaml :as yaml])
(:import
(java.io File)
- (java.nio.file Files Path)))
+ (java.nio.file Files FileVisitOption Path Paths)))
(set! *warn-on-reflection* true)
@@ -65,12 +64,6 @@
[]
(:path (plugins-dir-info)))
-(defn- extract-system-modules! []
- (when (io/resource "modules")
- (let [plugins-path (plugins-dir)]
- (u.files/with-open-path-to-resource [modules-path "modules"]
- (u.files/copy-files! modules-path plugins-path)))))
-
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | loading/initializing plugins |
;;; +----------------------------------------------------------------------------------------------------------------+
@@ -115,40 +108,36 @@
"spark-deps.jar is no longer needed by Metabase 0.32.0+. You can delete it from the plugins directory.")))]
path))
-(when (not config/is-prod?)
- (defn- load-local-plugin-manifest! [^Path path]
- (some-> (slurp (str path)) yaml/parse-string plugins.init/init-plugin-with-info!))
-
- (defn- driver-manifest-paths
- "Return a sequence of [[java.io.File]] paths for `metabase-plugin.yaml` plugin manifests for drivers on the classpath."
- []
- ;; only include plugin manifests if they're on the system classpath.
- (concat
- (for [^File file (classpath/system-classpath)
- :when (and (.isDirectory file)
- (not (.isHidden file))
- (str/includes? (str file) "modules/drivers")
- (or (str/ends-with? (str file) "resources")
- (str/ends-with? (str file) "resources-ee")))
- :let [manifest-file (io/file file "metabase-plugin.yaml")]
- :when (.exists manifest-file)]
- manifest-file)
- ;; for hacking on 3rd-party drivers locally: set
- ;; `-Dmb.dev.additional.driver.manifest.paths=/path/to/whatever/metabase-plugin.yaml` or
- ;; `MB_DEV_ADDITIONAL_DRIVER_MANIFEST_PATHS=...` to have that plugin manifest get loaded during startup. Specify
- ;; multiple plugin manifests by comma-separating them.
- (when-let [additional-paths (env/env :mb-dev-additional-driver-manifest-paths)]
- (map u.files/get-path (str/split additional-paths #",")))))
-
- (defn- load-local-plugin-manifests!
- "Load local plugin manifest files when not running in a production mode, to simulate what would happen when loading those
- same plugins from the uberjar. This is needed because some plugin manifests define driver methods and the like that
- aren't defined elsewhere."
- []
- ;; TODO - this should probably do an actual search in case we ever add any additional directories
- (doseq [manifest-path (driver-manifest-paths)]
- (log/infof "Loading local plugin manifest at %s" (str manifest-path))
- (load-local-plugin-manifest! manifest-path))))
+(defn- load-plugin-manifest! [path]
+ (some-> (slurp (str path)) yaml/parse-string plugins.init/init-plugin-with-info!))
+
+(defn- bundled-driver-manifest-paths
+ "Return a sequence of paths (URIs when running from a jar, Paths in dev) for `metabase-plugin.yaml` plugin manifests
+ for drivers on the classpath. Driver manifests live at `metabase/<driver-name>/metabase-plugin.yaml` on the classpath."
+ []
+ (if (u.files/running-from-jar?)
+ (u.files/find-in-current-jar "glob:/metabase/*/metabase-plugin.yaml")
+ (let [matcher (.getPathMatcher (java.nio.file.FileSystems/getDefault) "glob:**/metabase/*/metabase-plugin.yaml")]
+ (for [^File f (classpath/system-classpath)
+ :when (.isDirectory f)
+ founds [(-> (.toURI f)
+ (Paths/get)
+ (Files/find 3
+ (fn [path _attr] (.matches matcher path))
+ (make-array FileVisitOption 0))
+ (.collect (java.util.stream.Collectors/toList)))]
+ f founds]
+ f))))
+
+(defn- load-bundled-driver-plugin-manifests!
+ "Find and load driver plugin manifests from the classpath. In the uberjar, driver classes are flattened directly into
+ the jar (rather than shipped as nested JARs in a `modules/` directory) and their manifests live at
+ `metabase/<driver>/metabase-plugin.yaml`. In dev, the same manifests are found via the driver resource directories on
+ the classpath."
+ []
+ (doseq [manifest-path (bundled-driver-manifest-paths)]
+ (log/infof "Loading driver plugin manifest at %s" (str manifest-path))
+ (load-plugin-manifest! manifest-path)))
(defn- has-manifest? ^Boolean [^Path path]
(boolean (u.files/file-exists-in-archive? path "metabase-plugin.yaml")))
@@ -166,29 +155,29 @@
(log/errorf e "Failed to initialize plugin %s" (.getFileName path))))))
(defn- load! []
+ ;; Load any user-supplied plugin JARs from the plugins directory (e.g. Oracle JDBC driver).
+ ;; System/bundled drivers are no longer extracted to disk — they are flattened into the uberjar
+ ;; and their manifests are loaded from the classpath via load-driver-plugin-manifests! below.
(log/infof "Loading plugins in %s..." (str (plugins-dir)))
- (extract-system-modules!)
(let [paths (plugins-paths)]
(init-plugins! paths))
- (when (not config/is-prod?)
- (load-local-plugin-manifests!)))
+ (load-bundled-driver-plugin-manifests!))
(defonce ^:private loaded? (atom false))
(defn load-plugins!
- "Load Metabase plugins. The are JARs shipped as part of Metabase itself, under the `resources/modules` directory (the
- source for these JARs is under the `modules` directory); and others manually added by users to the Metabase plugins
- directory, which defaults to `./plugins`.
+ "Load Metabase plugins. Bundled drivers are compiled directly into the uberjar and their plugin manifests are
+ discovered on the classpath at `metabase/<driver>/metabase-plugin.yaml`. User-supplied plugins (e.g. Oracle JDBC
+ driver JARs) are loaded from the plugins directory, which defaults to `./plugins`.
When loading plugins, Metabase performs the following steps:
* Metabase creates the plugins directory if it does not already exist.
- * Any plugins that are shipped as part of Metabase itself are extracted from the Metabase uberjar (or `resources`
- directory when running with the Clojure CLI) into the plugins directory.
* Each JAR in the plugins directory that *does not* include a Metabase plugin manifest is added to the classpath.
* For JARs that include a Metabase plugin manifest (a `metabase-plugin.yaml` file), a lazy-loading Metabase driver
is registered; when the driver is initialized (automatically, when certain methods are called) the JAR is added
- to the classpath and the driver namespace is loaded
+ to the classpath and the driver namespace is loaded.
+ * Bundled driver plugin manifests are loaded from the classpath (not from disk).
This function will only perform loading steps the first time it is called — it is safe to call this function more
than once."
diff --git a/src/metabase/util/files.clj b/src/metabase/util/files.clj
index 0c41718ae12..519ee72516c 100644
--- a/src/metabase/util/files.clj
+++ b/src/metabase/util/files.clj
@@ -15,8 +15,8 @@
(:import
(java.io FileNotFoundException)
(java.net URL)
- (java.nio.file CopyOption Files FileSystem FileSystemAlreadyExistsException FileSystems
- LinkOption OpenOption Path Paths StandardCopyOption)
+ (java.nio.file CopyOption Files FileSystem FileSystemAlreadyExistsException
+ FileSystems LinkOption OpenOption Path Paths StandardCopyOption)
(java.nio.file.attribute FileAttribute)
(java.util Collections)
(java.util.zip ZipInputStream)))
@@ -183,3 +183,55 @@
"Returns a java.nio.file.Path"
[path]
(fs/relativize (fs/absolutize ".") path))
+
+(defn running-from-jar?
+ "Returns true iff we are running from a jar.
+
+ .getResource will return a java.net.URL, and those start with \"jar:\" if and only if the app is running from a jar.
+
+ More info: https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/lang/Thread.html"
+ []
+ (= "jar" (.. (Thread/currentThread)
+ getContextClassLoader
+ (getResource ".keep-me")
+ getProtocol)))
+
+(defn get-jar-path
+ "Returns the path to the currently running jar file.
+
+ More info: https://stackoverflow.com/questions/320542/how-to-get-the-path-of-a-running-jar-file"
+ []
+ (assert (running-from-jar?) "Can only get-jar-path when running from a jar.")
+ (-> (class {})
+ (.getProtectionDomain)
+ (.getCodeSource)
+ (.getLocation)
+ (.toURI) ;; avoid problems with special characters in path.
+ (.getPath)))
+
+(defn find-in-current-jar
+ "Find matching files in the current jar. See
+ https://docs.oracle.com/javase/8/docs/api/java/nio/file/FileSystem.html#getPathMatcher-java.lang.String- to
+ understand the syntax of [[pattern]]. Motivating use case is:
+
+ `(u.files/find-in-current-jar \"glob:/metabase/*/metabase-plugin.yaml\")`
+
+ to find the plugin manifests in the jar."
+ [pattern]
+ (let [jar-url (URL. (str "jar:file:" (get-jar-path) "!/"))]
+ (with-open [fs (jar-file-system-from-url jar-url)]
+ (let [matcher (.getPathMatcher fs pattern)
+ root (.getPath fs "/" (into-array String []))
+ files (atom [])]
+ (Files/walkFileTree root
+ (proxy [java.nio.file.SimpleFileVisitor] []
+ (preVisitDirectory [dir _attrs]
+ (if (or (str/starts-with? (str dir) "/metabase")
+ (= (str "/") (str dir)))
+ java.nio.file.FileVisitResult/CONTINUE
+ java.nio.file.FileVisitResult/SKIP_SUBTREE))
+ (visitFile [file _attrs]
+ (when (.matches matcher file)
+ (swap! files conj file))
+ java.nio.file.FileVisitResult/CONTINUE)))
+ (into [] (map #(.toUri ^Path %) @files))))))