Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

docs - consolidate doc generation (#84079)

Jeff Bruemmer32b7d05ddea5
diff --git a/.github/scripts/check-preresolve-aliases.sh b/.github/scripts/check-preresolve-aliases.sh
index 4828d656947..8487626abb7 100755
--- a/.github/scripts/check-preresolve-aliases.sh
+++ b/.github/scripts/check-preresolve-aliases.sh
@@ -59,14 +59,16 @@ scan() { # scan <edition>: dependency root and aliases used by .github Clojure i
| sed -e 's/:$//' -e 's#^#.|#'
}
-scan_project_tests() { # scan_project_tests <function> <dependency-root>
- # Mage shell commands are vectors split across lines, so flatten each function body before
- # extracting its alias.
- sed -n "/(defn- $1 /,/^(defn/p" mage/src/mage/project_tests.clj \
- | tr '\n' ' ' \
+mage_aliases() { # mage_aliases <dependency-root>: aliases of the `"clojure"` commands in Mage source on stdin
+ # Mage shell commands are vectors split across lines, so flatten the source before extracting aliases.
+ tr '\n' ' ' \
| grep -oE '"clojure"[[:space:]]+("-P"[[:space:]]+)?"-[XMATP]?:[A-Za-z0-9:_./-]+' \
| grep -oE ':[A-Za-z0-9:_./-]+' \
- | sed -e 's/:$//' -e "s#^#$2|#"
+ | sed -e 's/:$//' -e "s#^#$1|#"
+}
+
+scan_project_tests() { # scan_project_tests <function> <dependency-root>
+ sed -n "/(defn- $1 /,/^(defn/p" mage/src/mage/project_tests.clj | mage_aliases "$2"
}
# Check each scan separately. One scan still finding commands would otherwise conceal that another has
@@ -79,11 +81,13 @@ require_scan() { # require_scan <what> <result>
workflow_checks=$( { scan ee; scan oss; } || true)
clojure_checks=$(scan_project_tests run-clojure-checks! . || true)
migration_checks=$(scan_project_tests run-migration-checks! bin/lint-migrations-file || true)
+docs_checks=$(mage_aliases . < mage/src/mage/generate_docs.clj || true)
require_scan ".github/" "$workflow_checks"
require_scan "run-clojure-checks! (mage/src/mage/project_tests.clj)" "$clojure_checks"
require_scan "run-migration-checks! (mage/src/mage/project_tests.clj)" "$migration_checks"
+require_scan "mage/src/mage/generate_docs.clj" "$docs_checks"
-scanned="$(printf '%s\n' "$workflow_checks" "$clojure_checks" "$migration_checks" | sort -u)"
+scanned="$(printf '%s\n' "$workflow_checks" "$clojure_checks" "$migration_checks" "$docs_checks" | sort -u)"
found="$( {
printf '%s\n' "$scanned"
diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml
index 279a6789661..f5a00d633f3 100644
--- a/.github/workflows/cache-warm.yml
+++ b/.github/workflows/cache-warm.yml
@@ -127,7 +127,7 @@ jobs:
":dev:drivers:build:build-dev:build-test:ci" # build-scripts.yml
":test" # build-scripts.yml (bin/build unit tests)
":ee:drivers:load-namespaces" # backend.yml (load-namespaces)
- ":ee:doc" # docs-generate-base.yml
+ ":ee:doc" # mage generate-docs (docs-generate-base.yml)
":kondo" # resolve-backport-conflicts.yml (replace-deps)
":run:ee" # cross-branch-migration-test
":dev:ee:migrate" # cross-branch-migration-test
diff --git a/.github/workflows/docs-generate-base.yml b/.github/workflows/docs-generate-base.yml
index a300ad6cb45..96956c18c88 100644
--- a/.github/workflows/docs-generate-base.yml
+++ b/.github/workflows/docs-generate-base.yml
@@ -30,90 +30,27 @@ jobs:
token: ${{ secrets.automation_token }}
ref: ${{ inputs.target_branch }}
+ # Run mage from this workflow's ref (master), so target branches don't need each change to it.
+ # Only `docs/` gets committed, so this checkout stays out of the PR.
+ - name: Checkout docs generator
+ uses: actions/checkout@v7
+ with:
+ path: docs-generator
+ sparse-checkout: |
+ /bb.edn
+ /bin/mage
+ /bin/mage.bb
+ /mage/
+ sparse-checkout-cone-mode: false
+
- name: Prepare backend environment
uses: ./.github/actions/prepare-backend
- name: Prepare frontend environment
uses: ./.github/actions/prepare-frontend
- - name: Set up Babashka
- uses: turtlequeue/setup-babashka@d78ec6570aea3b614bc695cafcceb82eb45c2af9 # v1.8.0
- with:
- babashka-version: 1.12.197
-
- - name: Generate environment variables documentation
- id: generate_env_docs
- run: |
- set -e # Exit on any error
- # Ensure we're on the target branch for documentation generation
- git checkout ${{ inputs.target_branch }}
- echo "Generating environment variables documentation from branch: ${{ inputs.target_branch }}"
- clojure -M:ee:doc environment-variables-documentation
- continue-on-error: false
-
- - name: Generate config template documentation
- id: generate_config_docs
- run: |
- set -e # Exit on any error
- echo "Generating config template documentation..."
- clojure -M:ee:doc config-template
- continue-on-error: false
-
- - name: Generate API documentation
- id: generate_api_docs
- run: |
- set -e # Exit on any error
- echo "Generating API documentation..."
- clojure -M:ee:doc api-documentation
- continue-on-error: false
-
- - name: Generate command documentation
- id: generate_command_docs
- run: |
- set -e # Exit on any error
- echo "Generating API documentation..."
- clojure -M:ee:doc command-documentation
- continue-on-error: false
-
- - name: Generate AI provider documentation
- id: generate_ai_provider_docs
- # The generator only exists on branches that ship the multi-provider registry (master / 64+), but
- # this reusable workflow always runs from master, so skip the step on release branches without it.
- if: hashFiles('src/metabase/cmd/ai_provider_dox.clj') != ''
- run: |
- set -e # Exit on any error
- echo "Generating AI provider documentation..."
- clojure -M:ee:doc ai-providers-documentation
- continue-on-error: false
-
- - name: Generate MCP tool documentation
- id: generate_mcp_tool_docs
- # Same reason as the AI provider step: release branches without the generator would fail the command.
- if: hashFiles('src/metabase/cmd/mcp_tools_dox.clj') != ''
- run: |
- set -e # Exit on any error
- echo "Generating MCP tool documentation..."
- clojure -M:ee:doc mcp-tools-documentation
- continue-on-error: false
-
- - name: Generate usage analytics documentation
- run: ./bin/generate-usage-analytics-docs.bb
-
- - name: Generate embedding SDK documentation
- id: generate_embedding_sdk_docs
- run: |
- set -e # Exit on any error
- echo "Generating embedding SDK documentation..."
- bun run embedding-sdk:docs:generate
- continue-on-error: false
-
- - name: Generate embedding EAJS documentation
- id: generate_embedding_eajs_docs
- run: |
- set -e # Exit on any error
- echo "Generating embedding EAJS documentation..."
- bun run embedding-eajs:docs:generate
- continue-on-error: false
+ - name: Generate documentation
+ run: docs-generator/bin/mage generate-docs --dir "$GITHUB_WORKSPACE"
- name: Check for changes
id: check_changes
@@ -179,17 +116,8 @@ jobs:
$CHANGES_SUMMARY
$([ "$TOTAL_CHANGES" -gt 10 ] && echo "- ... and $((TOTAL_CHANGES - 10)) more files")
- ## Generation commands
- Documentation was generated using:
- - \`clojure -M:ee:doc environment-variables-documentation\`
- - \`clojure -M:ee:doc config-template\`
- - \`clojure -M:ee:doc api-documentation\`
- - \`clojure -M:ee:doc command-documentation\`
- - \`clojure -M:ee:doc ai-providers-documentation\`
- - \`clojure -M:ee:doc mcp-tools-documentation\`
- - \`./bin/generate-usage-analytics-docs.bb\`
- - \`bun run embedding-sdk:docs:generate\`
- - \`bun run embedding-eajs:docs:generate\`
+ ## Generation command
+ Documentation was generated using \`./bin/mage generate-docs\`.
Please review the changes and merge if they look correct." \
--label "Type:Documentation" \
diff --git a/.github/workflows/docs-release-generate.yml b/.github/workflows/docs-release-generate.yml
index f14d9b943e9..cd2c8f8a94a 100644
--- a/.github/workflows/docs-release-generate.yml
+++ b/.github/workflows/docs-release-generate.yml
@@ -6,17 +6,21 @@ on:
workflow_dispatch:
inputs:
target_branch:
- description: "Target branch for documentation update"
+ description: "Target branch for documentation update. Leave empty for every release branch below."
required: false
- default: "release-x.64.x"
+ default: ""
type: string
jobs:
update-release-docs:
if: ${{ github.event_name != 'schedule' || github.repository == 'metabase/metabase' }}
- name: Call documentation update for release branch
+ name: Call documentation update for ${{ matrix.target_branch }}
+ strategy:
+ fail-fast: false
+ matrix:
+ target_branch: ${{ fromJSON(inputs.target_branch && format('["{0}"]', inputs.target_branch) || '["release-x.64.x", "release-x.65.x"]') }}
uses: ./.github/workflows/docs-generate-base.yml
with:
- target_branch: ${{ inputs.target_branch || 'release-x.64.x' }}
+ target_branch: ${{ matrix.target_branch }}
secrets:
automation_token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
diff --git a/bb.edn b/bb.edn
index 42b1f6f1679..3c1f142d9fa 100644
--- a/bb.edn
+++ b/bb.edn
@@ -400,6 +400,24 @@
:requires [[mage.cursive-resolve-as :as cursive-resolve-as]]
:task (task! (cursive-resolve-as/do-it))}
+ generate-docs
+ {:doc "Regenerate the docs that are built from source. The backend suite runs every Clojure generator in one JVM."
+ :examples [["./bin/mage generate-docs" "Regenerate every generated doc"]
+ ["./bin/mage generate-docs backend" "Environment variables, config template, API, AI providers, MCP tools, and CLI commands"]
+ ["./bin/mage generate-docs usage-analytics" "Usage analytics reference"]
+ ["./bin/mage generate-docs embedding-sdk" "Embedding SDK reference (builds the SDK package first)"]
+ ["./bin/mage generate-docs embedding-eajs" "Modular embedding (EAJS) reference"]]
+ :options [["-d" "--dir DIR" "Absolute path of the Metabase checkout to generate the docs in. Defaults to this checkout."]]
+ :arg-schema [:or
+ [:tuple]
+ [:tuple [:enum {:name "suite"}
+ "backend"
+ "usage-analytics"
+ "embedding-sdk"
+ "embedding-eajs"]]]
+ :requires [[mage.generate-docs :as generate-docs]]
+ :task (task! (generate-docs/run! (first (:arguments parsed)) (:options parsed)))}
+
analytics:setup
{:doc "Set up usage analytics development mode"
:examples [["./bin/mage analytics:setup" "Enable usage analytics dev mode and import content"]]
diff --git a/deps.edn b/deps.edn
index ce192baabfb..4ca191bea1a 100644
--- a/deps.edn
+++ b/deps.edn
@@ -429,6 +429,10 @@
;; as some default values differ between dev and prod
;;
;; clojure -M:ee:doc api-documentation
+ ;;
+ ;; or run every documentation command in one JVM
+ ;;
+ ;; clojure -M:ee:doc all-documentation
:doc
{:main-opts ["-m" "metabase.core.bootstrap"]
:jvm-opts ["-Dmb.run.mode=prod"
diff --git a/docs/developers-guide/docs.md b/docs/developers-guide/docs.md
index 69e7a09c86b..af86dd2fb51 100644
--- a/docs/developers-guide/docs.md
+++ b/docs/developers-guide/docs.md
@@ -28,6 +28,30 @@ bun run lint-docs-links
You can view both commands in the [package.json](https://github.com/metabase/metabase/blob/master/package.json) file under `scripts`.
+## Regenerate docs built from source
+
+Some docs pages are generated from the source code. A scheduled GitHub workflow regenerates them every weekday and opens a PR with any changes.
+
+To regenerate them yourself, first install the [prerequisites for building Metabase](./build.md#install-the-prerequisites): Java, the Clojure CLI, Node.js, and Bun. You don't need to install Babashka, since `./bin/mage` installs it for you. Then install the frontend dependencies:
+
+```
+bun install
+```
+
+Then run:
+
+```
+./bin/mage generate-docs
+```
+
+To regenerate one group of pages, pass its suite name. For example:
+
+```
+./bin/mage generate-docs backend
+```
+
+To list the suites, run `./bin/mage generate-docs -h`. If any suite fails, the last line of output lists it.
+
## Updating API docs
To update an API endpoint description, you'll need to edit the comment in the [source code for that endpoint](https://github.com/metabase/metabase/tree/master/src/metabase/api).
@@ -35,7 +59,7 @@ To update an API endpoint description, you'll need to edit the comment in the [s
To bring your changes into `docs/latest/api-documentation`, you'll need to open a separate PR. Check out a new branch from the current release branch, and run:
```
-clojure -M:ee:run api-documentation
+clojure -M:ee:doc api-documentation
```
## Style guide
diff --git a/docs/installation-and-operation/commands.md b/docs/installation-and-operation/commands.md
index ae35aec99b0..140483d48f8 100644
--- a/docs/installation-and-operation/commands.md
+++ b/docs/installation-and-operation/commands.md
@@ -21,6 +21,10 @@ Metabase will print out the help text for available commands.
Generates a markdown file listing the AI providers Metabase can connect to, the credentials each one needs, and the models each one offers. This is written to a file called `docs/ai/providers.md`.
+## `all-documentation`
+
+Runs every documentation command in one process.
+
## `api-documentation`
Generate an HTML file and a JSON file for Scalar docs for the Metabase API.
diff --git a/mage/src/mage/generate_docs.clj b/mage/src/mage/generate_docs.clj
new file mode 100644
index 00000000000..76857dfd0db
--- /dev/null
+++ b/mage/src/mage/generate_docs.clj
@@ -0,0 +1,62 @@
+(ns mage.generate-docs
+ "Regenerates the docs that are built from source: environment variables, the config template, the API, AI providers,
+ MCP tools, CLI commands, the usage analytics reference, and the embedding SDK and EAJS references."
+ (:refer-clojure :exclude [run!])
+ (:require
+ [babashka.fs :as fs]
+ [clojure.string :as str]
+ [mage.shell :as shell]
+ [mage.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+;; .github/scripts/check-preresolve-aliases.sh reads the `clojure` alias out of this file.
+;; CI runs this list from master on each target branch, so every target branch needs each command.
+(def ^:private suites
+ "Each suite and its command, in the order the bare `generate-docs` command runs them."
+ [;; One JVM runs every Clojure generator, so Metabase's namespaces load once rather than once per document.
+ ["backend" ["clojure" "-M:ee:doc" "all-documentation"]]
+ ;; `./bin/mage` installs `bin/bb` in its own checkout only, not in the `--dir` checkout.
+ ["usage-analytics" [(str (fs/path u/project-root-directory "bin" "bb")) "bin/generate-usage-analytics-docs.bb"]]
+ ["embedding-sdk" ["bun" "run" "embedding-sdk:docs:generate"]]
+ ["embedding-eajs" ["bun" "run" "embedding-eajs:docs:generate"]]])
+
+(def ^:private suite->command (into {} suites))
+
+(def suite-names
+ "Every suite name, in the order the bare `generate-docs` command runs them."
+ (mapv first suites))
+
+(defn- run-suite!
+ "Run one suite and return its exit code.
+ A command that cannot be started, or that `sh` times out, counts as a failure rather than aborting the run."
+ [sh suite]
+ (println "Generating" suite "docs")
+ (let [exit (try
+ (:exit (apply sh (suite->command suite)))
+ (catch Exception e
+ (println "Could not generate" suite "docs --" (ex-message e))
+ 1))]
+ (when (zero? exit)
+ (println "Generated" suite "docs"))
+ exit))
+
+(defn run-suites!
+ "Run every suite in `names`, in order, with `sh`, a [[mage.shell/sh*]]-compatible function, and return the names of
+ the suites that failed. A failing suite does not stop the others."
+ [sh names]
+ (let [failed (into [] (remove #(zero? (run-suite! sh %))) names)]
+ (when (seq failed)
+ (println "Failed:" (str/join ", " failed)))
+ failed))
+
+(defn run!
+ "Generate the docs for `suite`, or for every suite when `suite` is nil.
+ The suites run in the checkout at `dir`, or in this checkout when `dir` is nil.
+ Exits nonzero when any suite fails."
+ [suite {:keys [dir]}]
+ (let [sh (if dir
+ (partial shell/sh* {:dir dir})
+ shell/sh*)]
+ (when (seq (run-suites! sh (if suite [suite] suite-names)))
+ (u/exit 1))))
diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.clj
index 81b0d93c1bf..43f91770a1b 100644
--- a/mage/test/mage/core_test.clj
+++ b/mage/test/mage/core_test.clj
@@ -17,6 +17,7 @@
[mage.bot.repl-eval-test]
[mage.doctor-test]
[mage.fix-unused-requires-test]
+ [mage.generate-docs-test]
[mage.kondo-ratchet-test]
[mage.merge-kondo-ratchets-test]
[mage.merge-yaml-migrations-test]
diff --git a/mage/test/mage/generate_docs_test.clj b/mage/test/mage/generate_docs_test.clj
new file mode 100644
index 00000000000..780f0286a72
--- /dev/null
+++ b/mage/test/mage/generate_docs_test.clj
@@ -0,0 +1,78 @@
+(ns mage.generate-docs-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing]]
+ [mage.generate-docs :as generate-docs]
+ [mage.shell :as shell]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private backend-command ["clojure" "-M:ee:doc" "all-documentation"])
+
+(defn- fake-sh
+ "A [[mage.shell/sh*]] stand-in.
+ Records each command in `calls` and exits with the code `exits` maps that command to, or 0.
+ An exception in `exits` is thrown instead, like a command that never started."
+ [calls exits]
+ (fn [& command]
+ (let [exit (get exits (vec command) 0)]
+ (swap! calls conj (vec command))
+ (when (instance? Exception exit)
+ (throw exit))
+ {:exit exit, :out [], :err []})))
+
+(defn- run-suites!
+ "Run `suite-names` through [[generate-docs/run-suites!]] with the given exit codes.
+ Returns the failed suites, the recorded commands, and everything printed."
+ [exits suite-names]
+ (let [calls (atom [])
+ out (java.io.StringWriter.)
+ failed (binding [*out* out]
+ (generate-docs/run-suites! (fake-sh calls exits) suite-names))]
+ {:failed failed, :calls @calls, :out (str out)}))
+
+(deftest every-suite-runs-test
+ (let [{:keys [failed calls out]} (run-suites! {} generate-docs/suite-names)]
+ (testing "every suite runs once, in order"
+ (is (= (map @#'generate-docs/suite->command generate-docs/suite-names) calls)))
+ (testing "each suite reports that it finished"
+ (doseq [suite generate-docs/suite-names]
+ (is (str/includes? out (str "Generated " suite " docs")))))
+ (is (= [] failed))))
+
+(deftest failed-suite-still-runs-later-suites-test
+ (let [{:keys [failed calls out]} (run-suites! {backend-command 1
+ ["bun" "run" "embedding-sdk:docs:generate"] 1}
+ generate-docs/suite-names)]
+ (testing "every suite runs"
+ (is (= 4 (count calls))))
+ (testing "the failed suites are reported, in suite order"
+ (is (= ["backend" "embedding-sdk"] failed))
+ (is (str/includes? out "Failed: backend, embedding-sdk")))))
+
+(deftest suite-that-cannot-start-is-a-failure-test
+ (let [{:keys [failed calls out]} (run-suites! {["bun" "run" "embedding-eajs:docs:generate"]
+ (ex-info "bun: command not found" {})}
+ ["embedding-eajs" "backend"])]
+ (testing "the other suite still runs"
+ (is (= 2 (count calls))))
+ (testing "the suite that could not run is reported as a failure"
+ (is (= ["embedding-eajs"] failed))
+ (is (str/includes? out "Could not generate embedding-eajs docs -- bun: command not found")))))
+
+(deftest run!-test
+ (testing "with a suite, runs only that suite and exits nonzero when it fails"
+ (let [calls (atom [])]
+ (with-redefs [shell/sh* (fake-sh calls {backend-command 1})]
+ (is (= 1 (try
+ (with-out-str (generate-docs/run! "backend" {}))
+ nil
+ (catch clojure.lang.ExceptionInfo e
+ (:babashka/exit (ex-data e)))))))
+ (is (= [backend-command] @calls))))
+ (testing "with `:dir`, runs every suite in that checkout"
+ (let [calls (atom [])]
+ (with-redefs [shell/sh* (fake-sh calls {})]
+ (with-out-str (generate-docs/run! nil {:dir "/tmp/release-checkout"})))
+ (is (= (repeat (count generate-docs/suite-names) {:dir "/tmp/release-checkout"})
+ (map first @calls))))))
diff --git a/src/metabase/cmd/core.clj b/src/metabase/cmd/core.clj
index 394d56534a1..1a139e294ea 100644
--- a/src/metabase/cmd/core.clj
+++ b/src/metabase/cmd/core.clj
@@ -187,6 +187,23 @@
(classloader/require 'metabase.cmd.command-dox)
((resolve 'metabase.cmd.command-dox/generate-dox!)))
+(def ^:private doc-generators
+ "The commands that `all-documentation` runs, in order.
+ Commands that load only their own namespaces run first. The two settings generators load every Metabase namespace,
+ so they run last and cannot change the output of the others."
+ [#'mcp-tools-documentation
+ #'ai-providers-documentation
+ #'api-documentation
+ #'command-documentation
+ #'config-template
+ #'environment-variables-documentation])
+
+(defn ^:command all-documentation
+ "Runs every documentation command in one process."
+ []
+ (doseq [generator doc-generators]
+ (generator)))
+
(defn ^:command driver-methods
"Print a list of all multimethods available for a driver to implement, optionally with their docstrings."
([]
diff --git a/test/metabase/cmd/core_test.clj b/test/metabase/cmd/core_test.clj
index 0a471c4f000..c86f1be8c90 100644
--- a/test/metabase/cmd/core_test.clj
+++ b/test/metabase/cmd/core_test.clj
@@ -1,5 +1,6 @@
(ns metabase.cmd.core-test
(:require
+ [clojure.string :as str]
[clojure.test :refer [are deftest is testing]]
[metabase.cmd.core :as cmd]
[metabase.test.util.dynamic-redefs :as dynamic-redefs]))
@@ -71,3 +72,13 @@
["--full-stacktrace"]
{:full-stacktrace true}))))
+
+(deftest ^:parallel all-documentation-test
+ (testing "`all-documentation` runs every `*-documentation` command"
+ (let [generators (set @#'cmd/doc-generators)]
+ (is (empty? (for [[symb varr] (ns-interns 'metabase.cmd.core)
+ :when (and (:command (meta varr))
+ (str/ends-with? (name symb) "-documentation")
+ (not= symb 'all-documentation)
+ (not (generators varr)))]
+ symb))))))