The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
handle duplicate copies of gson on classpath (#74685)
dpsuttonad0c7fb28cdb
diff --git a/bin/build/src/build/uberjar.clj b/bin/build/src/build/uberjar.cljindex 5f380631bb8..4ae1b8dff8a 100644--- a/bin/build/src/build/uberjar.clj+++ b/bin/build/src/build/uberjar.clj@@ -1,12 +1,14 @@(ns build.uberjar(:require[clojure.java.io :as io]+ [clojure.string :as str][clojure.tools.build.api :as b][clojure.tools.build.util.zip :as build.zip][clojure.tools.namespace.dependency :as ns.deps][clojure.tools.namespace.find :as ns.find][clojure.tools.namespace.parse :as ns.parse][metabuild-common.core :as u]+ [metabuild-common.misc :as misc][org.corfield.log4j2-conflict-handler :refer [log4j2-conflict-handler]])(:import(java.io File OutputStream)@@ -167,13 +169,45 @@#"META-INF/license.*"#"META-INF/LICENSE.*"])+(defn- prefer-lib+ "Returns a conflict handler fn that ensures `preferred` lib's classes always win.+ The returned fn writes when the incoming class is from `preferred`, skips otherwise."+ [preferred]+ (fn prefer-lib' [{:keys [lib path in]}]+ (when (= lib preferred)+ {:write {path {:stream in}}})))++;; hive-jdbc bundles javax.activation classes with package-private visibility on LogSupport.+;; When these overwrite jakarta.activation's public versions, javax.mail (postal) fails with+;; IllegalAccessError.+(def ^:private activation-conflict-handler+ (let [from-com-sun-activation (prefer-lib 'com.sun.activation/jakarta.activation)]+ {"com/sun/activation/.*" from-com-sun-activation+ "javax/activation/.*" from-com-sun-activation}))++;; vertica-jdbc bundles unshaded gson 2.8.9. When these overwrite the pinned 2.12.1,+;; BigQuery crashes with NoSuchMethodError on JsonWriter.value(float). See #73736.+(def ^:private gson-conflict-handler+ {"com/google/gson/.*" (prefer-lib 'com.google.code.gson/gson)})++;; avatica (Hive transitive dep) bundles the entire SLF4J API unshaded.+(def ^:private slf4j-conflict-handler+ {"org/slf4j/.*" (prefer-lib 'org.slf4j/slf4j-api)})++(def conflict-handlers+ "Merged conflict handlers for the uberjar build. Handles Log4j2 plugin merging,+ jakarta.activation class visibility, gson version pinning, and SLF4J API."+ (merge log4j2-conflict-handler+ activation-conflict-handler+ gson-conflict-handler+ slf4j-conflict-handler))+(defn- create-uberjar! [basis](u/step "Create uberjar"(with-duration-ms [duration-ms](b/uber {:class-dir class-dir:uber-file uberjar-filename- ;; merge Log4j2Plugins.dat files. (#50721)- :conflict-handlers log4j2-conflict-handler+ :conflict-handlers conflict-handlers:basis basis:exclude dependency-ignore-patterns})(u/announce "Created uberjar in %.1f seconds." (/ duration-ms 1000.0)))))@@ -209,8 +243,8 @@:when (.isFile f)](let [rel-path (.toString (.relativize (.toPath src-dir) (.toPath f)))target (u/get-path-in-filesystem fs rel-path)]- (Files/createDirectories (.getParent target) (into-array java.nio.file.attribute.FileAttribute []))- (Files/copy (.toPath f) target (into-array java.nio.file.CopyOption [])))))))))+ (Files/createDirectories (.getParent target) (misc/varargs java.nio.file.attribute.FileAttribute))+ (Files/copy (.toPath f) target (misc/varargs java.nio.file.CopyOption)))))))))(defn update-manifest!"Start a build step that updates the manifest.@@ -241,3 +275,42 @@(add-non-aot-driver-sources!)(update-manifest!))(u/announce "Built %s in %.1f seconds." uberjar-filename (/ duration-ms 1000.0)))))++(defn detect-class-conflicts+ "Run `b/uber` against `basis` (no AOT, no resources) and return a seq of+ `{:path ... :lib ...}` for every `.class` file conflict not already handled+ by our conflict handlers."+ [basis]+ (let [conflicts (atom [])]+ (clean!)+ (b/uber {:class-dir class-dir+ :uber-file uberjar-filename+ :conflict-handlers (merge conflict-handlers+ {:default (fn [{:keys [lib path]}]+ (when (str/ends-with? path ".class")+ (swap! conflicts conj {:path path :lib lib}))+ nil)})+ :basis basis+ :exclude dependency-ignore-patterns})+ @conflicts))++(defn audit-conflicts+ "Build a bare uberjar (no AOT, no resources) and report all class file conflicts.+ Useful for detecting vendored/unshaded dependencies in fat JARs.++ clojure -X:build:build/uberjar build.uberjar/audit-conflicts+ clojure -X:build:build/uberjar build.uberjar/audit-conflicts :edition :ee"+ [{:keys [edition], :or {edition :ee}}]+ (u/step (format "Audit %s uberjar class file conflicts" edition)+ (let [basis (create-basis edition)+ conflicts (detect-class-conflicts basis)]+ (when (seq conflicts)+ (u/announce "=== %d class file conflicts detected ===" (count conflicts))+ (let [report-file "target/conflict-report.txt"]+ (spit report-file+ (str/join "\n"+ (for [[path libs] (->> conflicts+ (group-by :path)+ (sort-by key))]+ (format "%s — %s" path (str/join ", " (map :lib libs))))))+ (u/announce "Conflict report written to %s" report-file))))))diff --git a/bin/build/test/build/uberjar_test.clj b/bin/build/test/build/uberjar_test.cljnew file mode 100644index 00000000000..6cceba50cf6--- /dev/null+++ b/bin/build/test/build/uberjar_test.clj@@ -0,0 +1,59 @@+(ns build.uberjar-test+ (:require+ [build.uberjar :as uberjar]+ [clojure.test :refer [deftest is testing]]))++(set! *warn-on-reflection* true)++(def ^:private basis+ "EE basis — includes all drivers, which is where the worst conflicts live."+ (delay (#'uberjar/create-basis :ee)))++(defn- conflicting-prefixes+ "Return Java package paths that have class file conflicts.+ e.g. `com/google/gson/Gson.class` → `com/google/gson`"+ [conflicts]+ (into #{}+ (keep (fn [{:keys [path]}]+ (let [last-slash (.lastIndexOf ^String path "/")]+ (when (pos? last-slash)+ (subs path 0 last-slash)))))+ conflicts))++;; Known conflicting packages. The test asserts on *package prefix* (stable) rather than+;; *lib* (order-dependent — the conflict handler only sees the second writer).+;; As we fix these, remove entries — the goal is to shrink this to #{}.+(def ^:private known-conflicting-prefixes+ '#{"com/microsoft/schemas" ;; poi-ooxml vs poi-ooxml-lite — same version, XML schema overlap+ "org/openxmlformats/schemas" ;; poi-ooxml vs poi-ooxml-lite — same version+ "org/apache/poi/schemas" ;; poi-ooxml vs poi-ooxml-lite — same version+ "org/w3/x2000" ;; poi-ooxml vs poi-ooxml-lite — XML digital signature schemas+ "org/etsi/uri" ;; poi-ooxml vs poi-ooxml-lite — digital signature schemas+ "jakarta/servlet" ;; jetty-servlet vs jakarta.servlet-api — same API version+ "javax/annotation" ;; jsr250-api vs jsr305 — annotation-only JARs+ "net/jcip/annotations" ;; jcip-annotations vs stephenc jcip-annotations — same lib, two Maven coords+ "io/netty/buffer" ;; databricks-jdbc-thin bundles custom Arrow netty buffers+ "org/apache/calcite/avatica" ;; avatica vs avatica-core — Hive transitive dep+ ;; org/slf4j — handled by slf4j-conflict-handler (prefers org.slf4j/slf4j-api)+ "org/apache/hadoop" ;; hadoop-common single-class overlap+ "org/apache/hive"}) ;; hive-common single-class overlap++(defn- prefix-matches?+ "True if `prefix` equals or is under any of the known prefixes."+ [prefix]+ (some (fn [known]+ (or (= prefix known)+ (.startsWith ^String prefix (str known "/"))))+ known-conflicting-prefixes))++(deftest class-file-conflicts-test+ (testing "No unexpected class file conflicts in the EE uberjar"+ ;; Takes ~2 minutes — runs b/uber without AOT or resources+ (let [conflicts (uberjar/detect-class-conflicts @basis)+ prefixes (conflicting-prefixes conflicts)+ unexpected (sort (remove prefix-matches? prefixes))]+ (is (empty? unexpected)+ (str "Unexpected class file conflicts in packages:\n"+ (pr-str unexpected)+ "\nIf benign, add to known-conflicting-prefixes with a comment. "+ "If dangerous, add a conflict handler in build.uberjar.")))))diff --git a/modules/drivers/databricks/deps.edn b/modules/drivers/databricks/deps.ednindex 1aba2d34c6c..12f5ed69292 100644--- a/modules/drivers/databricks/deps.edn+++ b/modules/drivers/databricks/deps.edn@@ -4,9 +4,8 @@:deps{at.yawk.lz4/lz4-java {:mvn/version "1.10.4"} ; pin newer version because version included in JDBC driver has security warningscom.databricks/databricks-jdbc-thin {:mvn/version "3.3.1"- ;; lang3 is a dep in top-level deps.edn, so not actually used anyway; exclusion- ;; here fixes security warnings about version used in JDBC driver- :exclusions [org.apache.commons/commons-lang3]}+ :exclusions [org.apache.commons/commons-lang3 ;; dep in top-level deps.edn; exclusion fixes security warnings+ org.slf4j/slf4j-jdk14]} ;; conflicts with our log4j2 SLF4J provider;; pin Bouncy Castle explicitly so the bundled driver JAR uses the patched versionorg.bouncycastle/bcpkix-jdk18on {:mvn/version "1.84"}org.bouncycastle/bcprov-jdk18on {:mvn/version "1.84"}