Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Publish both uber-jar and thin jar for Athena JDBC driver (#72480)

Edward Pageta5a132c663e4
diff --git a/.github/workflows/athena.yml b/.github/workflows/athena.yml
index 08948f7f5e8..2121177cf83 100644
--- a/.github/workflows/athena.yml
+++ b/.github/workflows/athena.yml
@@ -7,6 +7,11 @@ on:
description: 'Athena JDBC driver version to publish'
required: true
type: string
+ force:
+ description: 'Re-deploy even if version already exists'
+ required: false
+ default: false
+ type: boolean
permissions:
id-token: write
@@ -33,7 +38,7 @@ jobs:
aws-region: ${{ vars.AWS_REGION }}
- name: Deploy Athena JDBC driver
- run: ./bin/athena.bb "${VERSION}" --deploy
+ run: ./bin/athena.bb "${VERSION}" --deploy ${{ github.event.inputs.force == 'true' && '--force' || '' }}
env:
VERSION: ${{ github.event.inputs.version }}
@@ -42,4 +47,9 @@ jobs:
VERSION: ${{ github.event.inputs.version }}
run: |
aws s3 ls s3://metabase-maven-downloads/com/metabase/athena-jdbc/"${VERSION}"/
- echo "Athena JDBC driver version ${VERSION} has been deployed"
+ aws s3 ls s3://metabase-maven-downloads/com/metabase/athena-jdbc-thin/"${VERSION}"/
+ echo "Verifying thin jar pom exists..."
+ aws s3 ls s3://metabase-maven-downloads/com/metabase/athena-jdbc-thin/"${VERSION}"/athena-jdbc-thin-"${VERSION}".pom
+ echo "Verifying athena-streaming..."
+ aws s3 ls s3://metabase-maven-downloads/com/metabase/athena-streaming/2.0/
+ echo "Athena JDBC driver version ${VERSION} has been deployed (uber-jar, thin jar, and athena-streaming)"
diff --git a/bin/athena.bb b/bin/athena.bb
index 72a58e0b403..af155f512e8 100755
--- a/bin/athena.bb
+++ b/bin/athena.bb
@@ -19,10 +19,14 @@
;;
;; Will download the 2.0.35 jar from amazon's release bucket, create and update the necessary maven repo files, then upload all those files to our maven repo.
;;
-;; It uses the jar build WITHOUT the AWS SDK.
+;; It publishes both the uber-jar (with-dependencies) as `athena-jdbc` and the thin jar as `athena-jdbc-thin`.
;;
;; You must have bb (babashka), aws (awscli), and shasum commands available. You must have aws credentials available, e.g. through aws configure
+(def s3-bucket "s3://metabase-maven-downloads/")
+(def maven-base-url "https://s3.amazonaws.com/metabase-maven-downloads/")
+(def athena-downloads-url "https://s3.amazonaws.com/athena-downloads/")
+
(defn find-pred [z p?]
(->> (iterate z/next z)
(take-while (every-pred some? (complement z/end?)))
@@ -46,15 +50,31 @@
(recur parent)
z)))
-(defn jar-file-name [version]
- (str "athena-jdbc-" version ".jar"))
+(defn jar-file-name [artifact-id version]
+ (str artifact-id "-" version ".jar"))
+
+(defn version-exists? [artifact-id version]
+ (let [metadata-url (str maven-base-url "com/metabase/" artifact-id "/maven-metadata.xml")]
+ (try
+ (let [existing (x/parse (io/reader metadata-url))
+ z (z/xml-zip existing)]
+ (boolean (find-pred z (every-pred (tag= :version)
+ (content= version)))))
+ (catch Exception _ false))))
-(defn create-new-metadata [version]
- (let [existing (x/parse (io/reader "https://s3.amazonaws.com/metabase-maven-downloads/com/metabase/athena-jdbc/maven-metadata.xml"))
- z (z/xml-zip existing)]
- (when (find-pred z (every-pred (tag= :version)
- (content= version)))
- (throw (Exception. "Version already exists in maven-metadata.xml")))
+(defn create-new-metadata [artifact-id version]
+ (let [metadata-url (str maven-base-url "com/metabase/" artifact-id "/maven-metadata.xml")
+ existing (try
+ (x/parse (io/reader metadata-url))
+ (catch Exception _
+ ;; No existing metadata — create fresh
+ (x/element :metadata {}
+ (x/element :groupId {} "com.metabase")
+ (x/element :artifactId {} artifact-id)
+ (x/element :versioning {}
+ (x/element :release {} version)
+ (x/element :versions {})))))
+ z (z/xml-zip existing)]
(let [indent (-> z (find-pred (tag= :versions)) z/down first)
indent (when (string? indent) indent)
xml (-> z
@@ -66,12 +86,12 @@
(z/down)
(z/replace version)
(z/root))]
- (println version "=> maven-metadata.xml")
+ (println version "=> maven-metadata.xml (" artifact-id ")")
(with-open [w (io/writer "maven-metadata.xml")]
(x/emit xml w)))))
-(defn download-latest [version]
- (let [base-url "https://s3.amazonaws.com/athena-downloads/"
+(defn download-uber-jar [version]
+ (let [base-url athena-downloads-url
downloads (x/parse (io/reader base-url))
z (z/xml-zip downloads)
jar-file (-> z
@@ -80,38 +100,98 @@
(content= #(str/ends-with? % (str version "-with-dependencies.jar")))))
z/node
:content
- str/join)]
- (println jar-file "=>" (jar-file-name version))
- (p/shell "curl --progress-bar -o " (jar-file-name version) (str base-url jar-file))))
+ str/join)
+ local-name (jar-file-name "athena-jdbc" version)]
+ (println jar-file "=>" local-name)
+ (p/shell "curl --progress-bar -o " local-name (str base-url jar-file))
+ local-name))
+
+(defn pom-file-name [artifact-id version]
+ (str artifact-id "-" version ".pom"))
+
+(defn download-lean-zip
+ "Downloads and extracts the lean zip, returning the zip-file path for cleanup."
+ [version]
+ (let [zip-url (str athena-downloads-url "drivers/JDBC/" version
+ "/athena-jdbc-" version "-lean-jar-and-separate-dependencies-jars.zip")
+ zip-file (str "athena-jdbc-thin-" version ".zip")]
+ (println zip-url "=>" zip-file)
+ (p/shell "curl --progress-bar -o" zip-file zip-url)
+ zip-file))
-(defn create-sha1 [version]
- (let [jar (jar-file-name version)
- sha1 (-> (p/shell {:out :string} "shasum -a 1" jar)
+(defn download-thin-jar [version]
+ (let [zip-file (download-lean-zip version)
+ local-name (jar-file-name "athena-jdbc-thin" version)
+ pom-name (pom-file-name "athena-jdbc-thin" version)]
+ ;; Extract the thin jar and pom.xml from the zip
+ (p/shell "unzip" "-o" "-j" zip-file (str "athena-jdbc-" version ".jar") "pom.xml" "-d" ".")
+ ;; Rename to our artifact names
+ (let [extracted (str "athena-jdbc-" version ".jar")]
+ (when (not= extracted local-name)
+ (.renameTo (io/file extracted) (io/file local-name))))
+ (.renameTo (io/file "pom.xml") (io/file pom-name))
+ (.delete (io/file zip-file))
+ local-name))
+
+(def athena-streaming-version "2.0")
+
+(defn download-athena-streaming [version]
+ (let [zip-file (download-lean-zip version)
+ src-name "AthenaStreamingJavaClient-2.0.jar"
+ local-name (jar-file-name "athena-streaming" athena-streaming-version)]
+ ;; Extract athena-streaming from runtime-dependencies/
+ (p/shell "unzip" "-o" "-j" zip-file (str "runtime-dependencies/" src-name) "-d" ".")
+ (.renameTo (io/file src-name) (io/file local-name))
+ (.delete (io/file zip-file))
+ local-name))
+
+(defn create-sha1 [fname]
+ (let [sha1 (-> (p/shell {:out :string} "shasum -a 1" fname)
:out
(str/split #"\s+")
first)]
- (println "sha1(jar) =>" (str jar ".sha1"))
- (with-open [w (io/writer (str jar ".sha1"))]
+ (println "sha1(jar) =>" (str fname ".sha1"))
+ (with-open [w (io/writer (str fname ".sha1"))]
(.write w sha1))))
-(defn copy-to-s3 [version]
- (let [root-dir "com/metabase/athena-jdbc/"
- files [{:remote-dir (str version "/") :fname (jar-file-name version)}
- {:remote-dir (str version "/") :fname (str (jar-file-name version) ".sha1")}
- {:remote-dir nil :fname "maven-metadata.xml"}]]
+(defn copy-to-s3 [artifact-id version jar-fname]
+ (let [root-dir (str "com/metabase/" artifact-id "/")
+ pom-name (pom-file-name artifact-id version)
+ files (cond-> [{:remote-dir (str version "/") :fname jar-fname}
+ {:remote-dir (str version "/") :fname (str jar-fname ".sha1")}]
+ (.exists (io/file "maven-metadata.xml"))
+ (conj {:remote-dir nil :fname "maven-metadata.xml"})
+ (.exists (io/file pom-name))
+ (conj {:remote-dir (str version "/") :fname pom-name}))]
(doseq [{:keys [remote-dir fname]} files
- :let [target (str "s3://metabase-maven-downloads/" root-dir remote-dir fname)]]
+ :let [target (str s3-bucket root-dir remote-dir fname)]]
(println fname "=>" target)
(p/shell "aws s3 cp" fname target))))
-(defn deploy [version deploy?]
- (create-new-metadata version)
- (download-latest version)
- (create-sha1 version)
- (when deploy?
- (copy-to-s3 version)))
+(defn deploy-artifact [artifact-id version download-fn deploy? force?]
+ (if (and (not force?) (version-exists? artifact-id version))
+ (println "Version" version "already exists for" artifact-id "— skipping (use --force to re-deploy)")
+ (do
+ (when-not (version-exists? artifact-id version)
+ (create-new-metadata artifact-id version))
+ (let [jar-fname (download-fn version)]
+ (create-sha1 jar-fname)
+ (when deploy?
+ (copy-to-s3 artifact-id version jar-fname))))))
+
+(defn deploy [version deploy? force?]
+ (println "=== Publishing uber-jar (athena-jdbc) ===")
+ (deploy-artifact "athena-jdbc" version download-uber-jar deploy? force?)
+ (println)
+ (println "=== Publishing thin jar (athena-jdbc-thin) ===")
+ (deploy-artifact "athena-jdbc-thin" version download-thin-jar deploy? force?)
+ (println)
+ (println "=== Publishing athena-streaming ===")
+ (deploy-artifact "athena-streaming" athena-streaming-version
+ (fn [_] (download-athena-streaming version))
+ deploy? force?))
(when (= *file* (System/getProperty "babashka.file"))
- (let [{:keys [args opts]} (cli/parse-args *command-line-args* {:coerce {:deploy :boolean}})]
+ (let [{:keys [args opts]} (cli/parse-args *command-line-args* {:coerce {:deploy :boolean :force :boolean}})]
(when-let [version (first args)]
- (deploy version (:deploy opts)))))
+ (deploy version (:deploy opts) (:force opts)))))