The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
docs - consolidate doc generation (#84079)
Jeff Bruemmer32b7d05ddea5
diff --git a/.github/scripts/check-preresolve-aliases.sh b/.github/scripts/check-preresolve-aliases.shindex 4828d656947..8487626abb7 100755--- a/.github/scripts/check-preresolve-aliases.sh+++ b/.github/scripts/check-preresolve-aliases.sh@@ -59,14 +59,16 @@ scan() { # scan <edition>: dependency root and aliases used by .github Clojure i| sed -e 's/:$//' -e 's#^#.|#'}-scan_project_tests() { # scan_project_tests <function> <dependency-root>- # Mage shell commands are vectors split across lines, so flatten each function body before- # extracting its alias.- sed -n "/(defn- $1 /,/^(defn/p" mage/src/mage/project_tests.clj \- | tr '\n' ' ' \+mage_aliases() { # mage_aliases <dependency-root>: aliases of the `"clojure"` commands in Mage source on stdin+ # Mage shell commands are vectors split across lines, so flatten the source before extracting aliases.+ tr '\n' ' ' \| grep -oE '"clojure"[[:space:]]+("-P"[[:space:]]+)?"-[XMATP]?:[A-Za-z0-9:_./-]+' \| grep -oE ':[A-Za-z0-9:_./-]+' \- | sed -e 's/:$//' -e "s#^#$2|#"+ | sed -e 's/:$//' -e "s#^#$1|#"+}++scan_project_tests() { # scan_project_tests <function> <dependency-root>+ sed -n "/(defn- $1 /,/^(defn/p" mage/src/mage/project_tests.clj | mage_aliases "$2"}# Check each scan separately. One scan still finding commands would otherwise conceal that another has@@ -79,11 +81,13 @@ require_scan() { # require_scan <what> <result>workflow_checks=$( { scan ee; scan oss; } || true)clojure_checks=$(scan_project_tests run-clojure-checks! . || true)migration_checks=$(scan_project_tests run-migration-checks! bin/lint-migrations-file || true)+docs_checks=$(mage_aliases . < mage/src/mage/generate_docs.clj || true)require_scan ".github/" "$workflow_checks"require_scan "run-clojure-checks! (mage/src/mage/project_tests.clj)" "$clojure_checks"require_scan "run-migration-checks! (mage/src/mage/project_tests.clj)" "$migration_checks"+require_scan "mage/src/mage/generate_docs.clj" "$docs_checks"-scanned="$(printf '%s\n' "$workflow_checks" "$clojure_checks" "$migration_checks" | sort -u)"+scanned="$(printf '%s\n' "$workflow_checks" "$clojure_checks" "$migration_checks" "$docs_checks" | sort -u)"found="$( {printf '%s\n' "$scanned"diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.ymlindex 279a6789661..f5a00d633f3 100644--- a/.github/workflows/cache-warm.yml+++ b/.github/workflows/cache-warm.yml@@ -127,7 +127,7 @@ jobs:":dev:drivers:build:build-dev:build-test:ci" # build-scripts.yml":test" # build-scripts.yml (bin/build unit tests)":ee:drivers:load-namespaces" # backend.yml (load-namespaces)- ":ee:doc" # docs-generate-base.yml+ ":ee:doc" # mage generate-docs (docs-generate-base.yml)":kondo" # resolve-backport-conflicts.yml (replace-deps)":run:ee" # cross-branch-migration-test":dev:ee:migrate" # cross-branch-migration-testdiff --git a/.github/workflows/docs-generate-base.yml b/.github/workflows/docs-generate-base.ymlindex a300ad6cb45..96956c18c88 100644--- a/.github/workflows/docs-generate-base.yml+++ b/.github/workflows/docs-generate-base.yml@@ -30,90 +30,27 @@ jobs:token: ${{ secrets.automation_token }}ref: ${{ inputs.target_branch }}+ # Run mage from this workflow's ref (master), so target branches don't need each change to it.+ # Only `docs/` gets committed, so this checkout stays out of the PR.+ - name: Checkout docs generator+ uses: actions/checkout@v7+ with:+ path: docs-generator+ sparse-checkout: |+ /bb.edn+ /bin/mage+ /bin/mage.bb+ /mage/+ sparse-checkout-cone-mode: false+- name: Prepare backend environmentuses: ./.github/actions/prepare-backend- name: Prepare frontend environmentuses: ./.github/actions/prepare-frontend- - name: Set up Babashka- uses: turtlequeue/setup-babashka@d78ec6570aea3b614bc695cafcceb82eb45c2af9 # v1.8.0- with:- babashka-version: 1.12.197-- - name: Generate environment variables documentation- id: generate_env_docs- run: |- set -e # Exit on any error- # Ensure we're on the target branch for documentation generation- git checkout ${{ inputs.target_branch }}- echo "Generating environment variables documentation from branch: ${{ inputs.target_branch }}"- clojure -M:ee:doc environment-variables-documentation- continue-on-error: false-- - name: Generate config template documentation- id: generate_config_docs- run: |- set -e # Exit on any error- echo "Generating config template documentation..."- clojure -M:ee:doc config-template- continue-on-error: false-- - name: Generate API documentation- id: generate_api_docs- run: |- set -e # Exit on any error- echo "Generating API documentation..."- clojure -M:ee:doc api-documentation- continue-on-error: false-- - name: Generate command documentation- id: generate_command_docs- run: |- set -e # Exit on any error- echo "Generating API documentation..."- clojure -M:ee:doc command-documentation- continue-on-error: false-- - name: Generate AI provider documentation- id: generate_ai_provider_docs- # The generator only exists on branches that ship the multi-provider registry (master / 64+), but- # this reusable workflow always runs from master, so skip the step on release branches without it.- if: hashFiles('src/metabase/cmd/ai_provider_dox.clj') != ''- run: |- set -e # Exit on any error- echo "Generating AI provider documentation..."- clojure -M:ee:doc ai-providers-documentation- continue-on-error: false-- - name: Generate MCP tool documentation- id: generate_mcp_tool_docs- # Same reason as the AI provider step: release branches without the generator would fail the command.- if: hashFiles('src/metabase/cmd/mcp_tools_dox.clj') != ''- run: |- set -e # Exit on any error- echo "Generating MCP tool documentation..."- clojure -M:ee:doc mcp-tools-documentation- continue-on-error: false-- - name: Generate usage analytics documentation- run: ./bin/generate-usage-analytics-docs.bb-- - name: Generate embedding SDK documentation- id: generate_embedding_sdk_docs- run: |- set -e # Exit on any error- echo "Generating embedding SDK documentation..."- bun run embedding-sdk:docs:generate- continue-on-error: false-- - name: Generate embedding EAJS documentation- id: generate_embedding_eajs_docs- run: |- set -e # Exit on any error- echo "Generating embedding EAJS documentation..."- bun run embedding-eajs:docs:generate- continue-on-error: false+ - name: Generate documentation+ run: docs-generator/bin/mage generate-docs --dir "$GITHUB_WORKSPACE"- name: Check for changesid: check_changes@@ -179,17 +116,8 @@ jobs:$CHANGES_SUMMARY$([ "$TOTAL_CHANGES" -gt 10 ] && echo "- ... and $((TOTAL_CHANGES - 10)) more files")- ## Generation commands- Documentation was generated using:- - \`clojure -M:ee:doc environment-variables-documentation\`- - \`clojure -M:ee:doc config-template\`- - \`clojure -M:ee:doc api-documentation\`- - \`clojure -M:ee:doc command-documentation\`- - \`clojure -M:ee:doc ai-providers-documentation\`- - \`clojure -M:ee:doc mcp-tools-documentation\`- - \`./bin/generate-usage-analytics-docs.bb\`- - \`bun run embedding-sdk:docs:generate\`- - \`bun run embedding-eajs:docs:generate\`+ ## Generation command+ Documentation was generated using \`./bin/mage generate-docs\`.Please review the changes and merge if they look correct." \--label "Type:Documentation" \diff --git a/.github/workflows/docs-release-generate.yml b/.github/workflows/docs-release-generate.ymlindex f14d9b943e9..cd2c8f8a94a 100644--- a/.github/workflows/docs-release-generate.yml+++ b/.github/workflows/docs-release-generate.yml@@ -6,17 +6,21 @@ on:workflow_dispatch:inputs:target_branch:- description: "Target branch for documentation update"+ description: "Target branch for documentation update. Leave empty for every release branch below."required: false- default: "release-x.64.x"+ default: ""type: stringjobs:update-release-docs:if: ${{ github.event_name != 'schedule' || github.repository == 'metabase/metabase' }}- name: Call documentation update for release branch+ name: Call documentation update for ${{ matrix.target_branch }}+ strategy:+ fail-fast: false+ matrix:+ target_branch: ${{ fromJSON(inputs.target_branch && format('["{0}"]', inputs.target_branch) || '["release-x.64.x", "release-x.65.x"]') }}uses: ./.github/workflows/docs-generate-base.ymlwith:- target_branch: ${{ inputs.target_branch || 'release-x.64.x' }}+ target_branch: ${{ matrix.target_branch }}secrets:automation_token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}diff --git a/bb.edn b/bb.ednindex 42b1f6f1679..3c1f142d9fa 100644--- a/bb.edn+++ b/bb.edn@@ -400,6 +400,24 @@:requires [[mage.cursive-resolve-as :as cursive-resolve-as]]:task (task! (cursive-resolve-as/do-it))}+ generate-docs+ {:doc "Regenerate the docs that are built from source. The backend suite runs every Clojure generator in one JVM."+ :examples [["./bin/mage generate-docs" "Regenerate every generated doc"]+ ["./bin/mage generate-docs backend" "Environment variables, config template, API, AI providers, MCP tools, and CLI commands"]+ ["./bin/mage generate-docs usage-analytics" "Usage analytics reference"]+ ["./bin/mage generate-docs embedding-sdk" "Embedding SDK reference (builds the SDK package first)"]+ ["./bin/mage generate-docs embedding-eajs" "Modular embedding (EAJS) reference"]]+ :options [["-d" "--dir DIR" "Absolute path of the Metabase checkout to generate the docs in. Defaults to this checkout."]]+ :arg-schema [:or+ [:tuple]+ [:tuple [:enum {:name "suite"}+ "backend"+ "usage-analytics"+ "embedding-sdk"+ "embedding-eajs"]]]+ :requires [[mage.generate-docs :as generate-docs]]+ :task (task! (generate-docs/run! (first (:arguments parsed)) (:options parsed)))}+analytics:setup{:doc "Set up usage analytics development mode":examples [["./bin/mage analytics:setup" "Enable usage analytics dev mode and import content"]]diff --git a/deps.edn b/deps.ednindex ce192baabfb..4ca191bea1a 100644--- a/deps.edn+++ b/deps.edn@@ -429,6 +429,10 @@;; as some default values differ between dev and prod;;;; clojure -M:ee:doc api-documentation+ ;;+ ;; or run every documentation command in one JVM+ ;;+ ;; clojure -M:ee:doc all-documentation:doc{:main-opts ["-m" "metabase.core.bootstrap"]:jvm-opts ["-Dmb.run.mode=prod"diff --git a/docs/developers-guide/docs.md b/docs/developers-guide/docs.mdindex 69e7a09c86b..af86dd2fb51 100644--- a/docs/developers-guide/docs.md+++ b/docs/developers-guide/docs.md@@ -28,6 +28,30 @@ bun run lint-docs-linksYou can view both commands in the [package.json](https://github.com/metabase/metabase/blob/master/package.json) file under `scripts`.+## Regenerate docs built from source++Some docs pages are generated from the source code. A scheduled GitHub workflow regenerates them every weekday and opens a PR with any changes.++To regenerate them yourself, first install the [prerequisites for building Metabase](./build.md#install-the-prerequisites): Java, the Clojure CLI, Node.js, and Bun. You don't need to install Babashka, since `./bin/mage` installs it for you. Then install the frontend dependencies:++```+bun install+```++Then run:++```+./bin/mage generate-docs+```++To regenerate one group of pages, pass its suite name. For example:++```+./bin/mage generate-docs backend+```++To list the suites, run `./bin/mage generate-docs -h`. If any suite fails, the last line of output lists it.+## Updating API docsTo update an API endpoint description, you'll need to edit the comment in the [source code for that endpoint](https://github.com/metabase/metabase/tree/master/src/metabase/api).@@ -35,7 +59,7 @@ To update an API endpoint description, you'll need to edit the comment in the [sTo bring your changes into `docs/latest/api-documentation`, you'll need to open a separate PR. Check out a new branch from the current release branch, and run:```-clojure -M:ee:run api-documentation+clojure -M:ee:doc api-documentation```## Style guidediff --git a/docs/installation-and-operation/commands.md b/docs/installation-and-operation/commands.mdindex ae35aec99b0..140483d48f8 100644--- a/docs/installation-and-operation/commands.md+++ b/docs/installation-and-operation/commands.md@@ -21,6 +21,10 @@ Metabase will print out the help text for available commands.Generates a markdown file listing the AI providers Metabase can connect to, the credentials each one needs, and the models each one offers. This is written to a file called `docs/ai/providers.md`.+## `all-documentation`++Runs every documentation command in one process.+## `api-documentation`Generate an HTML file and a JSON file for Scalar docs for the Metabase API.diff --git a/mage/src/mage/generate_docs.clj b/mage/src/mage/generate_docs.cljnew file mode 100644index 00000000000..76857dfd0db--- /dev/null+++ b/mage/src/mage/generate_docs.clj@@ -0,0 +1,62 @@+(ns mage.generate-docs+ "Regenerates the docs that are built from source: environment variables, the config template, the API, AI providers,+ MCP tools, CLI commands, the usage analytics reference, and the embedding SDK and EAJS references."+ (:refer-clojure :exclude [run!])+ (:require+ [babashka.fs :as fs]+ [clojure.string :as str]+ [mage.shell :as shell]+ [mage.util :as u]))++(set! *warn-on-reflection* true)++;; .github/scripts/check-preresolve-aliases.sh reads the `clojure` alias out of this file.+;; CI runs this list from master on each target branch, so every target branch needs each command.+(def ^:private suites+ "Each suite and its command, in the order the bare `generate-docs` command runs them."+ [;; One JVM runs every Clojure generator, so Metabase's namespaces load once rather than once per document.+ ["backend" ["clojure" "-M:ee:doc" "all-documentation"]]+ ;; `./bin/mage` installs `bin/bb` in its own checkout only, not in the `--dir` checkout.+ ["usage-analytics" [(str (fs/path u/project-root-directory "bin" "bb")) "bin/generate-usage-analytics-docs.bb"]]+ ["embedding-sdk" ["bun" "run" "embedding-sdk:docs:generate"]]+ ["embedding-eajs" ["bun" "run" "embedding-eajs:docs:generate"]]])++(def ^:private suite->command (into {} suites))++(def suite-names+ "Every suite name, in the order the bare `generate-docs` command runs them."+ (mapv first suites))++(defn- run-suite!+ "Run one suite and return its exit code.+ A command that cannot be started, or that `sh` times out, counts as a failure rather than aborting the run."+ [sh suite]+ (println "Generating" suite "docs")+ (let [exit (try+ (:exit (apply sh (suite->command suite)))+ (catch Exception e+ (println "Could not generate" suite "docs --" (ex-message e))+ 1))]+ (when (zero? exit)+ (println "Generated" suite "docs"))+ exit))++(defn run-suites!+ "Run every suite in `names`, in order, with `sh`, a [[mage.shell/sh*]]-compatible function, and return the names of+ the suites that failed. A failing suite does not stop the others."+ [sh names]+ (let [failed (into [] (remove #(zero? (run-suite! sh %))) names)]+ (when (seq failed)+ (println "Failed:" (str/join ", " failed)))+ failed))++(defn run!+ "Generate the docs for `suite`, or for every suite when `suite` is nil.+ The suites run in the checkout at `dir`, or in this checkout when `dir` is nil.+ Exits nonzero when any suite fails."+ [suite {:keys [dir]}]+ (let [sh (if dir+ (partial shell/sh* {:dir dir})+ shell/sh*)]+ (when (seq (run-suites! sh (if suite [suite] suite-names)))+ (u/exit 1))))diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.cljindex 81b0d93c1bf..43f91770a1b 100644--- a/mage/test/mage/core_test.clj+++ b/mage/test/mage/core_test.clj@@ -17,6 +17,7 @@[mage.bot.repl-eval-test][mage.doctor-test][mage.fix-unused-requires-test]+ [mage.generate-docs-test][mage.kondo-ratchet-test][mage.merge-kondo-ratchets-test][mage.merge-yaml-migrations-test]diff --git a/mage/test/mage/generate_docs_test.clj b/mage/test/mage/generate_docs_test.cljnew file mode 100644index 00000000000..780f0286a72--- /dev/null+++ b/mage/test/mage/generate_docs_test.clj@@ -0,0 +1,78 @@+(ns mage.generate-docs-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing]]+ [mage.generate-docs :as generate-docs]+ [mage.shell :as shell]))++(set! *warn-on-reflection* true)++(def ^:private backend-command ["clojure" "-M:ee:doc" "all-documentation"])++(defn- fake-sh+ "A [[mage.shell/sh*]] stand-in.+ Records each command in `calls` and exits with the code `exits` maps that command to, or 0.+ An exception in `exits` is thrown instead, like a command that never started."+ [calls exits]+ (fn [& command]+ (let [exit (get exits (vec command) 0)]+ (swap! calls conj (vec command))+ (when (instance? Exception exit)+ (throw exit))+ {:exit exit, :out [], :err []})))++(defn- run-suites!+ "Run `suite-names` through [[generate-docs/run-suites!]] with the given exit codes.+ Returns the failed suites, the recorded commands, and everything printed."+ [exits suite-names]+ (let [calls (atom [])+ out (java.io.StringWriter.)+ failed (binding [*out* out]+ (generate-docs/run-suites! (fake-sh calls exits) suite-names))]+ {:failed failed, :calls @calls, :out (str out)}))++(deftest every-suite-runs-test+ (let [{:keys [failed calls out]} (run-suites! {} generate-docs/suite-names)]+ (testing "every suite runs once, in order"+ (is (= (map @#'generate-docs/suite->command generate-docs/suite-names) calls)))+ (testing "each suite reports that it finished"+ (doseq [suite generate-docs/suite-names]+ (is (str/includes? out (str "Generated " suite " docs")))))+ (is (= [] failed))))++(deftest failed-suite-still-runs-later-suites-test+ (let [{:keys [failed calls out]} (run-suites! {backend-command 1+ ["bun" "run" "embedding-sdk:docs:generate"] 1}+ generate-docs/suite-names)]+ (testing "every suite runs"+ (is (= 4 (count calls))))+ (testing "the failed suites are reported, in suite order"+ (is (= ["backend" "embedding-sdk"] failed))+ (is (str/includes? out "Failed: backend, embedding-sdk")))))++(deftest suite-that-cannot-start-is-a-failure-test+ (let [{:keys [failed calls out]} (run-suites! {["bun" "run" "embedding-eajs:docs:generate"]+ (ex-info "bun: command not found" {})}+ ["embedding-eajs" "backend"])]+ (testing "the other suite still runs"+ (is (= 2 (count calls))))+ (testing "the suite that could not run is reported as a failure"+ (is (= ["embedding-eajs"] failed))+ (is (str/includes? out "Could not generate embedding-eajs docs -- bun: command not found")))))++(deftest run!-test+ (testing "with a suite, runs only that suite and exits nonzero when it fails"+ (let [calls (atom [])]+ (with-redefs [shell/sh* (fake-sh calls {backend-command 1})]+ (is (= 1 (try+ (with-out-str (generate-docs/run! "backend" {}))+ nil+ (catch clojure.lang.ExceptionInfo e+ (:babashka/exit (ex-data e)))))))+ (is (= [backend-command] @calls))))+ (testing "with `:dir`, runs every suite in that checkout"+ (let [calls (atom [])]+ (with-redefs [shell/sh* (fake-sh calls {})]+ (with-out-str (generate-docs/run! nil {:dir "/tmp/release-checkout"})))+ (is (= (repeat (count generate-docs/suite-names) {:dir "/tmp/release-checkout"})+ (map first @calls))))))diff --git a/src/metabase/cmd/core.clj b/src/metabase/cmd/core.cljindex 394d56534a1..1a139e294ea 100644--- a/src/metabase/cmd/core.clj+++ b/src/metabase/cmd/core.clj@@ -187,6 +187,23 @@(classloader/require 'metabase.cmd.command-dox)((resolve 'metabase.cmd.command-dox/generate-dox!)))+(def ^:private doc-generators+ "The commands that `all-documentation` runs, in order.+ Commands that load only their own namespaces run first. The two settings generators load every Metabase namespace,+ so they run last and cannot change the output of the others."+ [#'mcp-tools-documentation+ #'ai-providers-documentation+ #'api-documentation+ #'command-documentation+ #'config-template+ #'environment-variables-documentation])++(defn ^:command all-documentation+ "Runs every documentation command in one process."+ []+ (doseq [generator doc-generators]+ (generator)))+(defn ^:command driver-methods"Print a list of all multimethods available for a driver to implement, optionally with their docstrings."([]diff --git a/test/metabase/cmd/core_test.clj b/test/metabase/cmd/core_test.cljindex 0a471c4f000..c86f1be8c90 100644--- a/test/metabase/cmd/core_test.clj+++ b/test/metabase/cmd/core_test.clj@@ -1,5 +1,6 @@(ns metabase.cmd.core-test(:require+ [clojure.string :as str][clojure.test :refer [are deftest is testing]][metabase.cmd.core :as cmd][metabase.test.util.dynamic-redefs :as dynamic-redefs]))@@ -71,3 +72,13 @@["--full-stacktrace"]{:full-stacktrace true}))))++(deftest ^:parallel all-documentation-test+ (testing "`all-documentation` runs every `*-documentation` command"+ (let [generators (set @#'cmd/doc-generators)]+ (is (empty? (for [[symb varr] (ns-interns 'metabase.cmd.core)+ :when (and (:command (meta varr))+ (str/ends-with? (name symb) "-documentation")+ (not= symb 'all-documentation)+ (not (generators varr)))]+ symb))))))