The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Publish both uber-jar and thin jar for Athena JDBC driver (#72480)
Edward Pageta5a132c663e4
diff --git a/.github/workflows/athena.yml b/.github/workflows/athena.ymlindex 08948f7f5e8..2121177cf83 100644--- a/.github/workflows/athena.yml+++ b/.github/workflows/athena.yml@@ -7,6 +7,11 @@ on:description: 'Athena JDBC driver version to publish'required: truetype: string+ force:+ description: 'Re-deploy even if version already exists'+ required: false+ default: false+ type: booleanpermissions:id-token: write@@ -33,7 +38,7 @@ jobs:aws-region: ${{ vars.AWS_REGION }}- name: Deploy Athena JDBC driver- run: ./bin/athena.bb "${VERSION}" --deploy+ run: ./bin/athena.bb "${VERSION}" --deploy ${{ github.event.inputs.force == 'true' && '--force' || '' }}env:VERSION: ${{ github.event.inputs.version }}@@ -42,4 +47,9 @@ jobs:VERSION: ${{ github.event.inputs.version }}run: |aws s3 ls s3://metabase-maven-downloads/com/metabase/athena-jdbc/"${VERSION}"/- echo "Athena JDBC driver version ${VERSION} has been deployed"+ aws s3 ls s3://metabase-maven-downloads/com/metabase/athena-jdbc-thin/"${VERSION}"/+ echo "Verifying thin jar pom exists..."+ aws s3 ls s3://metabase-maven-downloads/com/metabase/athena-jdbc-thin/"${VERSION}"/athena-jdbc-thin-"${VERSION}".pom+ echo "Verifying athena-streaming..."+ aws s3 ls s3://metabase-maven-downloads/com/metabase/athena-streaming/2.0/+ echo "Athena JDBC driver version ${VERSION} has been deployed (uber-jar, thin jar, and athena-streaming)"diff --git a/bin/athena.bb b/bin/athena.bbindex 72a58e0b403..af155f512e8 100755--- a/bin/athena.bb+++ b/bin/athena.bb@@ -19,10 +19,14 @@;;;; Will download the 2.0.35 jar from amazon's release bucket, create and update the necessary maven repo files, then upload all those files to our maven repo.;;-;; It uses the jar build WITHOUT the AWS SDK.+;; It publishes both the uber-jar (with-dependencies) as `athena-jdbc` and the thin jar as `athena-jdbc-thin`.;;;; You must have bb (babashka), aws (awscli), and shasum commands available. You must have aws credentials available, e.g. through aws configure+(def s3-bucket "s3://metabase-maven-downloads/")+(def maven-base-url "https://s3.amazonaws.com/metabase-maven-downloads/")+(def athena-downloads-url "https://s3.amazonaws.com/athena-downloads/")+(defn find-pred [z p?](->> (iterate z/next z)(take-while (every-pred some? (complement z/end?)))@@ -46,15 +50,31 @@(recur parent)z)))-(defn jar-file-name [version]- (str "athena-jdbc-" version ".jar"))+(defn jar-file-name [artifact-id version]+ (str artifact-id "-" version ".jar"))++(defn version-exists? [artifact-id version]+ (let [metadata-url (str maven-base-url "com/metabase/" artifact-id "/maven-metadata.xml")]+ (try+ (let [existing (x/parse (io/reader metadata-url))+ z (z/xml-zip existing)]+ (boolean (find-pred z (every-pred (tag= :version)+ (content= version)))))+ (catch Exception _ false))))-(defn create-new-metadata [version]- (let [existing (x/parse (io/reader "https://s3.amazonaws.com/metabase-maven-downloads/com/metabase/athena-jdbc/maven-metadata.xml"))- z (z/xml-zip existing)]- (when (find-pred z (every-pred (tag= :version)- (content= version)))- (throw (Exception. "Version already exists in maven-metadata.xml")))+(defn create-new-metadata [artifact-id version]+ (let [metadata-url (str maven-base-url "com/metabase/" artifact-id "/maven-metadata.xml")+ existing (try+ (x/parse (io/reader metadata-url))+ (catch Exception _+ ;; No existing metadata — create fresh+ (x/element :metadata {}+ (x/element :groupId {} "com.metabase")+ (x/element :artifactId {} artifact-id)+ (x/element :versioning {}+ (x/element :release {} version)+ (x/element :versions {})))))+ z (z/xml-zip existing)](let [indent (-> z (find-pred (tag= :versions)) z/down first)indent (when (string? indent) indent)xml (-> z@@ -66,12 +86,12 @@(z/down)(z/replace version)(z/root))]- (println version "=> maven-metadata.xml")+ (println version "=> maven-metadata.xml (" artifact-id ")")(with-open [w (io/writer "maven-metadata.xml")](x/emit xml w)))))-(defn download-latest [version]- (let [base-url "https://s3.amazonaws.com/athena-downloads/"+(defn download-uber-jar [version]+ (let [base-url athena-downloads-urldownloads (x/parse (io/reader base-url))z (z/xml-zip downloads)jar-file (-> z@@ -80,38 +100,98 @@(content= #(str/ends-with? % (str version "-with-dependencies.jar")))))z/node:content- str/join)]- (println jar-file "=>" (jar-file-name version))- (p/shell "curl --progress-bar -o " (jar-file-name version) (str base-url jar-file))))+ str/join)+ local-name (jar-file-name "athena-jdbc" version)]+ (println jar-file "=>" local-name)+ (p/shell "curl --progress-bar -o " local-name (str base-url jar-file))+ local-name))++(defn pom-file-name [artifact-id version]+ (str artifact-id "-" version ".pom"))++(defn download-lean-zip+ "Downloads and extracts the lean zip, returning the zip-file path for cleanup."+ [version]+ (let [zip-url (str athena-downloads-url "drivers/JDBC/" version+ "/athena-jdbc-" version "-lean-jar-and-separate-dependencies-jars.zip")+ zip-file (str "athena-jdbc-thin-" version ".zip")]+ (println zip-url "=>" zip-file)+ (p/shell "curl --progress-bar -o" zip-file zip-url)+ zip-file))-(defn create-sha1 [version]- (let [jar (jar-file-name version)- sha1 (-> (p/shell {:out :string} "shasum -a 1" jar)+(defn download-thin-jar [version]+ (let [zip-file (download-lean-zip version)+ local-name (jar-file-name "athena-jdbc-thin" version)+ pom-name (pom-file-name "athena-jdbc-thin" version)]+ ;; Extract the thin jar and pom.xml from the zip+ (p/shell "unzip" "-o" "-j" zip-file (str "athena-jdbc-" version ".jar") "pom.xml" "-d" ".")+ ;; Rename to our artifact names+ (let [extracted (str "athena-jdbc-" version ".jar")]+ (when (not= extracted local-name)+ (.renameTo (io/file extracted) (io/file local-name))))+ (.renameTo (io/file "pom.xml") (io/file pom-name))+ (.delete (io/file zip-file))+ local-name))++(def athena-streaming-version "2.0")++(defn download-athena-streaming [version]+ (let [zip-file (download-lean-zip version)+ src-name "AthenaStreamingJavaClient-2.0.jar"+ local-name (jar-file-name "athena-streaming" athena-streaming-version)]+ ;; Extract athena-streaming from runtime-dependencies/+ (p/shell "unzip" "-o" "-j" zip-file (str "runtime-dependencies/" src-name) "-d" ".")+ (.renameTo (io/file src-name) (io/file local-name))+ (.delete (io/file zip-file))+ local-name))++(defn create-sha1 [fname]+ (let [sha1 (-> (p/shell {:out :string} "shasum -a 1" fname):out(str/split #"\s+")first)]- (println "sha1(jar) =>" (str jar ".sha1"))- (with-open [w (io/writer (str jar ".sha1"))]+ (println "sha1(jar) =>" (str fname ".sha1"))+ (with-open [w (io/writer (str fname ".sha1"))](.write w sha1))))-(defn copy-to-s3 [version]- (let [root-dir "com/metabase/athena-jdbc/"- files [{:remote-dir (str version "/") :fname (jar-file-name version)}- {:remote-dir (str version "/") :fname (str (jar-file-name version) ".sha1")}- {:remote-dir nil :fname "maven-metadata.xml"}]]+(defn copy-to-s3 [artifact-id version jar-fname]+ (let [root-dir (str "com/metabase/" artifact-id "/")+ pom-name (pom-file-name artifact-id version)+ files (cond-> [{:remote-dir (str version "/") :fname jar-fname}+ {:remote-dir (str version "/") :fname (str jar-fname ".sha1")}]+ (.exists (io/file "maven-metadata.xml"))+ (conj {:remote-dir nil :fname "maven-metadata.xml"})+ (.exists (io/file pom-name))+ (conj {:remote-dir (str version "/") :fname pom-name}))](doseq [{:keys [remote-dir fname]} files- :let [target (str "s3://metabase-maven-downloads/" root-dir remote-dir fname)]]+ :let [target (str s3-bucket root-dir remote-dir fname)]](println fname "=>" target)(p/shell "aws s3 cp" fname target))))-(defn deploy [version deploy?]- (create-new-metadata version)- (download-latest version)- (create-sha1 version)- (when deploy?- (copy-to-s3 version)))+(defn deploy-artifact [artifact-id version download-fn deploy? force?]+ (if (and (not force?) (version-exists? artifact-id version))+ (println "Version" version "already exists for" artifact-id "— skipping (use --force to re-deploy)")+ (do+ (when-not (version-exists? artifact-id version)+ (create-new-metadata artifact-id version))+ (let [jar-fname (download-fn version)]+ (create-sha1 jar-fname)+ (when deploy?+ (copy-to-s3 artifact-id version jar-fname))))))++(defn deploy [version deploy? force?]+ (println "=== Publishing uber-jar (athena-jdbc) ===")+ (deploy-artifact "athena-jdbc" version download-uber-jar deploy? force?)+ (println)+ (println "=== Publishing thin jar (athena-jdbc-thin) ===")+ (deploy-artifact "athena-jdbc-thin" version download-thin-jar deploy? force?)+ (println)+ (println "=== Publishing athena-streaming ===")+ (deploy-artifact "athena-streaming" athena-streaming-version+ (fn [_] (download-athena-streaming version))+ deploy? force?))(when (= *file* (System/getProperty "babashka.file"))- (let [{:keys [args opts]} (cli/parse-args *command-line-args* {:coerce {:deploy :boolean}})]+ (let [{:keys [args opts]} (cli/parse-args *command-line-args* {:coerce {:deploy :boolean :force :boolean}})](when-let [version (first args)]- (deploy version (:deploy opts)))))+ (deploy version (:deploy opts) (:force opts)))))