Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

handle duplicate copies of gson on classpath (#74685)

dpsuttonad0c7fb28cdb
diff --git a/bin/build/src/build/uberjar.clj b/bin/build/src/build/uberjar.clj
index 5f380631bb8..4ae1b8dff8a 100644
--- a/bin/build/src/build/uberjar.clj
+++ b/bin/build/src/build/uberjar.clj
@@ -1,12 +1,14 @@
(ns build.uberjar
(:require
[clojure.java.io :as io]
+ [clojure.string :as str]
[clojure.tools.build.api :as b]
[clojure.tools.build.util.zip :as build.zip]
[clojure.tools.namespace.dependency :as ns.deps]
[clojure.tools.namespace.find :as ns.find]
[clojure.tools.namespace.parse :as ns.parse]
[metabuild-common.core :as u]
+ [metabuild-common.misc :as misc]
[org.corfield.log4j2-conflict-handler :refer [log4j2-conflict-handler]])
(:import
(java.io File OutputStream)
@@ -167,13 +169,45 @@
#"META-INF/license.*"
#"META-INF/LICENSE.*"])
+(defn- prefer-lib
+ "Returns a conflict handler fn that ensures `preferred` lib's classes always win.
+ The returned fn writes when the incoming class is from `preferred`, skips otherwise."
+ [preferred]
+ (fn prefer-lib' [{:keys [lib path in]}]
+ (when (= lib preferred)
+ {:write {path {:stream in}}})))
+
+;; hive-jdbc bundles javax.activation classes with package-private visibility on LogSupport.
+;; When these overwrite jakarta.activation's public versions, javax.mail (postal) fails with
+;; IllegalAccessError.
+(def ^:private activation-conflict-handler
+ (let [from-com-sun-activation (prefer-lib 'com.sun.activation/jakarta.activation)]
+ {"com/sun/activation/.*" from-com-sun-activation
+ "javax/activation/.*" from-com-sun-activation}))
+
+;; vertica-jdbc bundles unshaded gson 2.8.9. When these overwrite the pinned 2.12.1,
+;; BigQuery crashes with NoSuchMethodError on JsonWriter.value(float). See #73736.
+(def ^:private gson-conflict-handler
+ {"com/google/gson/.*" (prefer-lib 'com.google.code.gson/gson)})
+
+;; avatica (Hive transitive dep) bundles the entire SLF4J API unshaded.
+(def ^:private slf4j-conflict-handler
+ {"org/slf4j/.*" (prefer-lib 'org.slf4j/slf4j-api)})
+
+(def conflict-handlers
+ "Merged conflict handlers for the uberjar build. Handles Log4j2 plugin merging,
+ jakarta.activation class visibility, gson version pinning, and SLF4J API."
+ (merge log4j2-conflict-handler
+ activation-conflict-handler
+ gson-conflict-handler
+ slf4j-conflict-handler))
+
(defn- create-uberjar! [basis]
(u/step "Create uberjar"
(with-duration-ms [duration-ms]
(b/uber {:class-dir class-dir
:uber-file uberjar-filename
- ;; merge Log4j2Plugins.dat files. (#50721)
- :conflict-handlers log4j2-conflict-handler
+ :conflict-handlers conflict-handlers
:basis basis
:exclude dependency-ignore-patterns})
(u/announce "Created uberjar in %.1f seconds." (/ duration-ms 1000.0)))))
@@ -209,8 +243,8 @@
:when (.isFile f)]
(let [rel-path (.toString (.relativize (.toPath src-dir) (.toPath f)))
target (u/get-path-in-filesystem fs rel-path)]
- (Files/createDirectories (.getParent target) (into-array java.nio.file.attribute.FileAttribute []))
- (Files/copy (.toPath f) target (into-array java.nio.file.CopyOption [])))))))))
+ (Files/createDirectories (.getParent target) (misc/varargs java.nio.file.attribute.FileAttribute))
+ (Files/copy (.toPath f) target (misc/varargs java.nio.file.CopyOption)))))))))
(defn update-manifest!
"Start a build step that updates the manifest.
@@ -241,3 +275,42 @@
(add-non-aot-driver-sources!)
(update-manifest!))
(u/announce "Built %s in %.1f seconds." uberjar-filename (/ duration-ms 1000.0)))))
+
+(defn detect-class-conflicts
+ "Run `b/uber` against `basis` (no AOT, no resources) and return a seq of
+ `{:path ... :lib ...}` for every `.class` file conflict not already handled
+ by our conflict handlers."
+ [basis]
+ (let [conflicts (atom [])]
+ (clean!)
+ (b/uber {:class-dir class-dir
+ :uber-file uberjar-filename
+ :conflict-handlers (merge conflict-handlers
+ {:default (fn [{:keys [lib path]}]
+ (when (str/ends-with? path ".class")
+ (swap! conflicts conj {:path path :lib lib}))
+ nil)})
+ :basis basis
+ :exclude dependency-ignore-patterns})
+ @conflicts))
+
+(defn audit-conflicts
+ "Build a bare uberjar (no AOT, no resources) and report all class file conflicts.
+ Useful for detecting vendored/unshaded dependencies in fat JARs.
+
+ clojure -X:build:build/uberjar build.uberjar/audit-conflicts
+ clojure -X:build:build/uberjar build.uberjar/audit-conflicts :edition :ee"
+ [{:keys [edition], :or {edition :ee}}]
+ (u/step (format "Audit %s uberjar class file conflicts" edition)
+ (let [basis (create-basis edition)
+ conflicts (detect-class-conflicts basis)]
+ (when (seq conflicts)
+ (u/announce "=== %d class file conflicts detected ===" (count conflicts))
+ (let [report-file "target/conflict-report.txt"]
+ (spit report-file
+ (str/join "\n"
+ (for [[path libs] (->> conflicts
+ (group-by :path)
+ (sort-by key))]
+ (format "%s — %s" path (str/join ", " (map :lib libs))))))
+ (u/announce "Conflict report written to %s" report-file))))))
diff --git a/bin/build/test/build/uberjar_test.clj b/bin/build/test/build/uberjar_test.clj
new file mode 100644
index 00000000000..6cceba50cf6
--- /dev/null
+++ b/bin/build/test/build/uberjar_test.clj
@@ -0,0 +1,59 @@
+(ns build.uberjar-test
+ (:require
+ [build.uberjar :as uberjar]
+ [clojure.test :refer [deftest is testing]]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private basis
+ "EE basis — includes all drivers, which is where the worst conflicts live."
+ (delay (#'uberjar/create-basis :ee)))
+
+(defn- conflicting-prefixes
+ "Return Java package paths that have class file conflicts.
+ e.g. `com/google/gson/Gson.class` → `com/google/gson`"
+ [conflicts]
+ (into #{}
+ (keep (fn [{:keys [path]}]
+ (let [last-slash (.lastIndexOf ^String path "/")]
+ (when (pos? last-slash)
+ (subs path 0 last-slash)))))
+ conflicts))
+
+;; Known conflicting packages. The test asserts on *package prefix* (stable) rather than
+;; *lib* (order-dependent — the conflict handler only sees the second writer).
+;; As we fix these, remove entries — the goal is to shrink this to #{}.
+(def ^:private known-conflicting-prefixes
+ '#{"com/microsoft/schemas" ;; poi-ooxml vs poi-ooxml-lite — same version, XML schema overlap
+ "org/openxmlformats/schemas" ;; poi-ooxml vs poi-ooxml-lite — same version
+ "org/apache/poi/schemas" ;; poi-ooxml vs poi-ooxml-lite — same version
+ "org/w3/x2000" ;; poi-ooxml vs poi-ooxml-lite — XML digital signature schemas
+ "org/etsi/uri" ;; poi-ooxml vs poi-ooxml-lite — digital signature schemas
+ "jakarta/servlet" ;; jetty-servlet vs jakarta.servlet-api — same API version
+ "javax/annotation" ;; jsr250-api vs jsr305 — annotation-only JARs
+ "net/jcip/annotations" ;; jcip-annotations vs stephenc jcip-annotations — same lib, two Maven coords
+ "io/netty/buffer" ;; databricks-jdbc-thin bundles custom Arrow netty buffers
+ "org/apache/calcite/avatica" ;; avatica vs avatica-core — Hive transitive dep
+ ;; org/slf4j — handled by slf4j-conflict-handler (prefers org.slf4j/slf4j-api)
+ "org/apache/hadoop" ;; hadoop-common single-class overlap
+ "org/apache/hive"}) ;; hive-common single-class overlap
+
+(defn- prefix-matches?
+ "True if `prefix` equals or is under any of the known prefixes."
+ [prefix]
+ (some (fn [known]
+ (or (= prefix known)
+ (.startsWith ^String prefix (str known "/"))))
+ known-conflicting-prefixes))
+
+(deftest class-file-conflicts-test
+ (testing "No unexpected class file conflicts in the EE uberjar"
+ ;; Takes ~2 minutes — runs b/uber without AOT or resources
+ (let [conflicts (uberjar/detect-class-conflicts @basis)
+ prefixes (conflicting-prefixes conflicts)
+ unexpected (sort (remove prefix-matches? prefixes))]
+ (is (empty? unexpected)
+ (str "Unexpected class file conflicts in packages:\n"
+ (pr-str unexpected)
+ "\nIf benign, add to known-conflicting-prefixes with a comment. "
+ "If dangerous, add a conflict handler in build.uberjar.")))))
diff --git a/modules/drivers/databricks/deps.edn b/modules/drivers/databricks/deps.edn
index 1aba2d34c6c..12f5ed69292 100644
--- a/modules/drivers/databricks/deps.edn
+++ b/modules/drivers/databricks/deps.edn
@@ -4,9 +4,8 @@
:deps
{at.yawk.lz4/lz4-java {:mvn/version "1.10.4"} ; pin newer version because version included in JDBC driver has security warnings
com.databricks/databricks-jdbc-thin {:mvn/version "3.3.1"
- ;; lang3 is a dep in top-level deps.edn, so not actually used anyway; exclusion
- ;; here fixes security warnings about version used in JDBC driver
- :exclusions [org.apache.commons/commons-lang3]}
+ :exclusions [org.apache.commons/commons-lang3 ;; dep in top-level deps.edn; exclusion fixes security warnings
+ org.slf4j/slf4j-jdk14]} ;; conflicts with our log4j2 SLF4J provider
;; pin Bouncy Castle explicitly so the bundled driver JAR uses the patched version
org.bouncycastle/bcpkix-jdk18on {:mvn/version "1.84"}
org.bouncycastle/bcprov-jdk18on {:mvn/version "1.84"}