The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add mage ci-report command for CI failure analysis (#68319)
bryan3b8e1ecaa4ee
diff --git a/.claude/commands/ci-report.md b/.claude/commands/ci-report.mdnew file mode 100644index 00000000000..3d8cf3a5914--- /dev/null+++ b/.claude/commands/ci-report.md@@ -0,0 +1,12 @@+Generate a CI failure report for PR $PR_NUM_OR_URL (or current branch if no argument given).++Run `./bin/mage ci-report $PR_NUM_OR_URL` to generate the report, then:++1. **Summarize the failures** - What tests failed and in which jobs?+2. **Categorize** - Are these real failures, flaky tests, or infrastructure issues?+3. **Recommend next steps**:+ - If flaky: suggest rerunning with `gh run rerun <run-id> --failed -R metabase/metabase`+ - If real: identify the root cause and suggest fixes+ - If infrastructure: note the issue type (timeout, OOM, network)++Focus on actionable insights. If all checks are passing, confirm that briefly.diff --git a/bb.edn b/bb.ednindex 1362aba7cdb..e0baa9b0dad 100644--- a/bb.edn+++ b/bb.edn@@ -283,6 +283,20 @@:requires [[mage.doctor :as doctor]]:task (task! (doctor/doctor!))}+ ci-report+ {:doc "Generate a CI failure report for a PR"+ :examples [["./bin/mage ci-report" "Check current branch, find its PR, and generate report"]+ ["./bin/mage ci-report 68292" "Generate report for PR #68292"]+ ["./bin/mage ci-report https://github.com/metabase/metabase/pull/68292" "Generate report from PR URL"]]+ :usage-fn (fn [_]+ (str "Fetches CI check status and failed job logs from GitHub Actions.\n"+ "Parses Trunk test reports to extract test failure details.\n\n"+ "With no arguments, looks up the PR for your current git branch.\n\n"+ "Requires GitHub CLI (gh) to be installed and authenticated.\n"+ "If you see 'gh: command not found' or auth errors, run: gh auth login"))+ :requires [[mage.ci-report :as ci-report]]+ :task (task! (ci-report/generate-report! parsed))}+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Private Tasks:;; These tasks are more for internal tooling, e.g. for use from git hooks etc.diff --git a/mage/src/mage/ci_report.clj b/mage/src/mage/ci_report.cljnew file mode 100644index 00000000000..ce15ac227c3--- /dev/null+++ b/mage/src/mage/ci_report.clj@@ -0,0 +1,361 @@+(ns mage.ci-report+ (:require+ [babashka.process :as p]+ [cheshire.core :as json]+ [clojure.string :as str]))++(def ^:private repo "metabase/metabase")++;;; Utilities++(defn- sh+ "Run shell command, return stdout or nil on error"+ [& args]+ (try+ (let [result (apply p/shell {:out :string :err :string :in nil} args)]+ (when (zero? (:exit result))+ (str/trim (:out result))))+ (catch Exception e+ (binding [*out* *err*]+ (println (format "Shell error: %s" (.getMessage e))))+ nil)))++(defn- gh-api+ "Call GitHub API via gh cli, return parsed JSON"+ [endpoint]+ (when-let [result (sh "gh" "api" endpoint)]+ (json/parse-string result true)))++(defn- strip-ansi+ "Remove ANSI escape codes from string"+ [s]+ (-> s+ (str/replace #"\x1b\[[0-9;]*m" "")+ (str/replace #"\[J" "")+ (str/replace #"\[K" "")))++(defn- log-progress [msg]+ (binding [*out* *err*]+ (println (str "\u001b[34m→\u001b[0m " msg))))++(defn- log-success [msg]+ (binding [*out* *err*]+ (println (str "\u001b[32m✓\u001b[0m " msg))))++;;; GitHub API++(defn- pr-info+ "Fetch PR metadata"+ [pr-number]+ (when-let [result (sh "gh" "pr" "view" (str pr-number) "-R" repo+ "--json" "headRefName,headRefOid,title,url")]+ (json/parse-string result true)))++(defn- pr-checks+ "Fetch PR check statuses"+ [pr-number]+ (when-let [result (sh "gh" "pr" "checks" (str pr-number) "-R" repo+ "--json" "name,state,link")]+ (json/parse-string result true)))++(defn- find-test-report-start+ "Find the line index where the test report section starts.+ Tries multiple markers for robustness against format changes."+ [lines]+ (or+ ;; Primary: 📚 Test Report header from trunk-analytics-cli+ (->> lines+ (map-indexed vector)+ (filter (fn [[_ line]] (str/includes? line "📚")))+ first+ first)+ ;; Fallback 1: Test summary line pattern (Total: X Pass: Y Fail: Z)+ (->> lines+ (map-indexed vector)+ (filter (fn [[_ line]] (re-find #"Total:\s*\d+\s+Pass:" line)))+ first+ first)+ ;; Fallback 2: Trunk.io links (appear in test failure reports)+ (->> lines+ (map-indexed vector)+ (filter (fn [[_ line]] (str/includes? line "trunk.io")))+ first+ first)))++(defn- extract-test-report-section+ "Extract the test report section from logs.+ For jobs with verbose cleanup output, the test report may be buried+ far from the end of the logs."+ [logs]+ (let [lines (str/split-lines logs)+ report-idx (find-test-report-start lines)]+ (if report-idx+ ;; Take 200 lines starting from the marker (report is typically <50 lines)+ (->> lines+ (drop report-idx)+ (take 200)+ (str/join "\n"))+ ;; No marker found - fall back to last 2000 lines+ (->> lines+ (take-last 2000)+ (str/join "\n")))))++(defn- job-logs+ "Fetch logs for a specific job ID"+ [job-id]+ (try+ (let [result (p/shell {:out :string :err :string :continue true}+ "timeout" "60" "gh" "api"+ (format "repos/%s/actions/jobs/%s/logs" repo job-id))]+ (when (zero? (:exit result))+ (extract-test-report-section (:out result))))+ (catch Exception _ nil)))++(defn- extract-job-id+ "Extract job ID from a check link URL"+ [link]+ (when link+ (second (re-find #"job/(\d+)" link))))++;;; Log Parsing++(defn- parse-trunk-report+ "Extract test failure info from Trunk test report section in logs"+ [logs]+ (let [clean-logs (strip-ansi logs)+ lines (str/split-lines clean-logs)+ ;; Find the test report section - look for 📚 emoji specifically+ in-report? (atom false)+ results (atom [])]+ (doseq [line lines]+ ;; Start at "📚 Test Report" (the emoji version, not config text)+ (when (str/includes? line "📚")+ (reset! in-report? true))+ ;; Stop at "Post job cleanup" or error marker+ (when (and @in-report?+ (or (str/includes? line "Post job cleanup")+ (str/includes? line "##[error]")))+ (reset! in-report? false))+ ;; Extract relevant info+ (when @in-report?+ (cond+ ;; Summary line: "Total: X Pass: Y Fail: Z"+ (re-find #"Total:.*Pass:.*Fail:" line)+ (let [clean-line (str/replace line #"^[0-9T:.Z-]+ +" "")]+ (swap! results conj {:type :summary :text (str/trim clean-line)}))+ ;; Failed count line (contains "test" and "failed" but not config text)+ (and (str/includes? line "failed")+ (str/includes? line "quarantined"))+ (let [clean-line (str/replace line #"^[0-9T:.Z-]+ +" "")]+ (swap! results conj {:type :failed-count :text (str/trim clean-line)}))+ ;; Package name with 📦+ (str/includes? line "📦")+ (when-let [pkg (re-find #"metabase[\w.-]+-test" line)]+ (swap! results conj {:type :package :text pkg}))+ ;; Test name (namespace-test/test-name) - must have a slash+ (and (re-find #"metabase[\w.-]+-test/" line)+ (not (str/includes? line "📦"))) ;; Not the package line+ (when-let [test-name (re-find #"metabase[\w.-]+-test/[\w-]+" line)]+ (swap! results conj {:type :test-name :text test-name}))+ ;; Trunk link+ (str/includes? line "trunk.io")+ (when-let [link (re-find #"https://app\.trunk\.io/[^\s]+" line)]+ (swap! results conj {:type :trunk-link :text link})))))+ @results))++(defn- format-trunk-report+ "Format parsed trunk report as markdown"+ [report-items]+ (when (seq report-items)+ (->> report-items+ (map (fn [{:keys [type text]}]+ (case type+ :summary text+ :failed-count text ;; Already has ❌ from the logs+ :package (str "📦 " text)+ :test-name (str " ❌ " text)+ :trunk-link (str " ⤷ " text)+ :quarantine text ;; Already has ⚠️ from the logs+ text)))+ (str/join "\n"))))++;;; Report Generation++(defn- categorize-checks+ "Categorize checks by state"+ [checks]+ {:failed (filter #(#{"FAILURE" "ERROR"} (:state %)) checks)+ :pending (filter #(#{"PENDING" "IN_PROGRESS" "QUEUED"} (:state %)) checks)+ :passed (filter #(= "SUCCESS" (:state %)) checks)})++(defn- fetch-failed-logs-parallel+ "Fetch logs for failed checks in parallel"+ [failed-checks]+ (let [job-ids (->> failed-checks+ (map :link)+ (map extract-job-id)+ (filter some?)+ distinct+ (take 15))] ; Limit to 15 jobs+ (when (seq job-ids)+ (log-progress (format " Fetching logs for %d failed job(s) in parallel..." (count job-ids)))+ (->> job-ids+ (pmap (fn [job-id]+ (when-let [logs (job-logs job-id)]+ [job-id logs])))+ (filter some?)+ (into {})))))++(defn- generate-report+ "Generate markdown report"+ [pr-number pr-info checks logs-by-job-id]+ (let [{:keys [failed pending passed]} (categorize-checks checks)+ branch (:headRefName pr-info)+ sha (subs (:headRefOid pr-info) 0 7)+ title (:title pr-info)+ pr-url (:url pr-info)]++ (println (format "# CI Status Report for PR #%s" pr-number))+ (println)+ (println (format "**Title:** %s" title))+ (println)+ (println (format "**Branch:** `%s` | **Commit:** `%s`" branch sha))+ (println)+ (println (format "**PR:** %s" pr-url))+ (println)++ ;; Summary table+ (println "## Summary")+ (println)+ (println "| Status | Count |")+ (println "|--------|-------|")+ (println (format "| ✅ Passed | %d |" (count passed)))+ (println (format "| ❌ Failed | %d |" (count failed)))+ (println (format "| ⏳ Pending | %d |" (count pending)))+ (println (format "| **Total** | **%d** |" (count checks)))+ (println)++ ;; Status message+ (cond+ (and (zero? (count failed)) (zero? (count pending)))+ (do (println "✅ **All checks passing!**")+ (println))++ (and (pos? (count pending)) (zero? (count failed)))+ (do (println (format "⏳ **CI is still running...** (%d checks pending)" (count pending)))+ (println))++ (pos? (count pending))+ (do (println (format "⚠️ **CI has failures and is still running** (%d failed, %d pending)"+ (count failed) (count pending)))+ (println)))++ (println "---")+ (println)++ ;; Failed checks detail+ (when (pos? (count failed))+ (println "## ❌ Failed Checks")+ (println)+ (doseq [check (sort-by :name failed)]+ (println (format "### %s" (:name check)))+ (println)+ (when-let [link (:link check)]+ (println (format "🔗 %s" link))+ (println))++ ;; Get logs for this check+ (let [job-id (extract-job-id (:link check))+ job-logs (get logs-by-job-id job-id)]+ (if job-logs+ (let [report (parse-trunk-report job-logs)+ formatted (format-trunk-report report)]+ (if (seq formatted)+ (do (println "**Test Failures:**")+ (println)+ (println "```")+ (println formatted)+ (println "```"))+ (println "_No test failures found in logs._")))+ (println "_Logs not yet available or run still in progress._")))+ (println)))++ ;; Pending checks+ (when (pos? (count pending))+ (println "## ⏳ Pending Checks")+ (println)+ (doseq [check (sort-by :name pending)]+ (println (format "- %s" (:name check))))+ (println))++ (println "---")+ (println)+ (println (format "_Generated by `bin/ci-failures.bb` at %s UTC_"+ (.format (java.time.LocalDateTime/now)+ (java.time.format.DateTimeFormatter/ofPattern "yyyy-MM-dd HH:mm:ss"))))))++;;; Main++(defn- get-current-branch-pr+ "Get PR number for current branch, or nil if none"+ []+ (when-let [result (sh "gh" "pr" "view" "--json" "number" "-q" ".number")]+ (when (re-matches #"\d+" result)+ result)))++(defn- parse-pr-arg+ "Parse PR argument - could be a number or a URL"+ [arg]+ (cond+ ;; Already a number+ (re-matches #"\d+" arg) arg+ ;; GitHub PR URL - extract the number+ (re-find #"github\.com/.+/pull/(\d+)" arg)+ (second (re-find #"github\.com/.+/pull/(\d+)" arg))+ ;; Otherwise return as-is and let gh handle it+ :else arg))++(defn generate-report! [{:keys [arguments]}]+ (let [pr-number (if (empty? arguments)+ ;; No args - try to get PR for current branch+ (do+ (log-progress "No PR specified, checking current branch...")+ (if-let [pr (get-current-branch-pr)]+ (do (log-success (format "Found PR #%s for current branch" pr))+ pr)+ (do+ (binding [*out* *err*]+ (println "Error: No PR found for current branch")+ (println "Usage: mage ci-report <pr-number or url>"))+ (System/exit 1))))+ ;; Parse the provided argument+ (parse-pr-arg (first arguments)))]+ (log-progress (format "Fetching PR #%s info..." pr-number))++ (let [pr-info (pr-info pr-number)]+ (when-not pr-info+ (binding [*out* *err*]+ (println (format "Error: Could not fetch PR #%s" pr-number)))+ (System/exit 1))++ (log-success (format "Found PR on branch: %s" (:headRefName pr-info)))+ (log-progress "Checking PR status...")++ (let [checks (pr-checks pr-number)+ {:keys [failed pending]} (categorize-checks checks)]++ (log-progress (format "Found %d failed, %d pending check(s)"+ (count failed) (count pending)))++ ;; Fetch logs for failed checks+ (log-progress "Fetching failed job logs...")+ (let [logs-by-job-id (if (pos? (count failed))+ (fetch-failed-logs-parallel failed)+ {})]+ (when (seq logs-by-job-id)+ (log-success "Retrieved logs"))++ (log-progress "Generating report...")+ (println)+ (generate-report pr-number pr-info checks logs-by-job-id)+ (log-success "Done!"))))))