Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

docs - add mage docs-preview (#85425)

Jeff Bruemmerdbd86b4960ec
diff --git a/.github/scripts/check-preresolve-aliases.sh b/.github/scripts/check-preresolve-aliases.sh
index 8487626abb7..44984dc8960 100755
--- a/.github/scripts/check-preresolve-aliases.sh
+++ b/.github/scripts/check-preresolve-aliases.sh
@@ -81,11 +81,11 @@ require_scan() { # require_scan <what> <result>
workflow_checks=$( { scan ee; scan oss; } || true)
clojure_checks=$(scan_project_tests run-clojure-checks! . || true)
migration_checks=$(scan_project_tests run-migration-checks! bin/lint-migrations-file || true)
-docs_checks=$(mage_aliases . < mage/src/mage/generate_docs.clj || true)
+docs_checks=$(mage_aliases . < mage/src/mage/docs_generate.clj || true)
require_scan ".github/" "$workflow_checks"
require_scan "run-clojure-checks! (mage/src/mage/project_tests.clj)" "$clojure_checks"
require_scan "run-migration-checks! (mage/src/mage/project_tests.clj)" "$migration_checks"
-require_scan "mage/src/mage/generate_docs.clj" "$docs_checks"
+require_scan "mage/src/mage/docs_generate.clj" "$docs_checks"
scanned="$(printf '%s\n' "$workflow_checks" "$clojure_checks" "$migration_checks" "$docs_checks" | sort -u)"
diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml
index fefca46fd4f..7e22320473f 100644
--- a/.github/workflows/cache-warm.yml
+++ b/.github/workflows/cache-warm.yml
@@ -128,7 +128,7 @@ jobs:
":dev:drivers:build:build-dev:build-test:ci" # build-scripts.yml
":test" # build-scripts.yml (bin/build unit tests)
":ee:drivers:load-namespaces" # backend.yml (load-namespaces)
- ":ee:doc" # mage generate-docs (docs-generate-base.yml)
+ ":ee:doc" # mage docs-generate (docs-generate-base.yml)
":kondo" # resolve-backport-conflicts.yml (replace-deps)
":run:ee" # cross-branch-migration-test
":dev:ee:migrate" # cross-branch-migration-test
diff --git a/.github/workflows/docs-generate-base.yml b/.github/workflows/docs-generate-base.yml
index 96956c18c88..622dd865693 100644
--- a/.github/workflows/docs-generate-base.yml
+++ b/.github/workflows/docs-generate-base.yml
@@ -50,7 +50,7 @@ jobs:
uses: ./.github/actions/prepare-frontend
- name: Generate documentation
- run: docs-generator/bin/mage generate-docs --dir "$GITHUB_WORKSPACE"
+ run: docs-generator/bin/mage docs-generate --dir "$GITHUB_WORKSPACE"
- name: Check for changes
id: check_changes
@@ -117,7 +117,7 @@ jobs:
$([ "$TOTAL_CHANGES" -gt 10 ] && echo "- ... and $((TOTAL_CHANGES - 10)) more files")
## Generation command
- Documentation was generated using \`./bin/mage generate-docs\`.
+ Documentation was generated using \`./bin/mage docs-generate\`.
Please review the changes and merge if they look correct." \
--label "Type:Documentation" \
diff --git a/bb.edn b/bb.edn
index fd6757cdb16..c30ea1d4c2d 100644
--- a/bb.edn
+++ b/bb.edn
@@ -122,6 +122,19 @@
:requires [[mage.start-maildev :as start-maildev]]
:task (task! (start-maildev/start-maildev!))}
+ docs-preview
+ {:doc "Preview the docs of a Metabase repo in your metabase-dot-com checkout, at http://localhost:4000/docs/latest/"
+ :examples [["./bin/mage docs-preview" "Preview the docs of this repo"]
+ ["./bin/mage docs-preview .claude/worktrees/my-feature" "Preview the docs of a worktree (a relative path starts at this repo)"]
+ ["./bin/mage docs-preview ~/code/other-metabase" "Preview the docs of another checkout"]
+ ["./bin/mage docs-preview --site ~/code/www" "Use the metabase-dot-com checkout at ~/code/www"]
+ ["METABASE_DOT_COM_DIR=~/code/www ./bin/mage docs-preview" "The same, from the env. Set it in your shell profile to use it always"]]
+ :requires [[mage.docs-preview :as docs-preview]]
+ :options [[nil "--site DIR" "The metabase-dot-com checkout. A relative path starts at this repo. Default: $METABASE_DOT_COM_DIR, then ~/metabase-dot-com"]
+ [nil "--no-open" "Do not open the browser"]]
+ :arg-schema [:or [:tuple] [:tuple [:string {:name "repo-path" :description "The Metabase repo or worktree with the docs. Default: this repo"}]]]
+ :task (task! (docs-preview/docs-preview! parsed))}
+
tail-d
{:doc "Tail the #d debug log in real time (see dev.debug). Asks the running REPL for the exact path; falls back to DEV_DEBUG_LOG or /tmp/<dir>_debug.log."
:examples [["./bin/mage tail-d" "tail the #d log, with bat syntax highlighting if available"]]
@@ -420,13 +433,13 @@
:requires [[mage.cursive-resolve-as :as cursive-resolve-as]]
:task (task! (cursive-resolve-as/do-it))}
- generate-docs
+ docs-generate
{:doc "Regenerate the docs that are built from source. The backend suite runs every Clojure generator in one JVM."
- :examples [["./bin/mage generate-docs" "Regenerate every generated doc"]
- ["./bin/mage generate-docs backend" "Environment variables, config template, API, AI providers, MCP tools, and CLI commands"]
- ["./bin/mage generate-docs usage-analytics" "Usage analytics reference"]
- ["./bin/mage generate-docs embedding-sdk" "Embedding SDK reference (builds the SDK package first)"]
- ["./bin/mage generate-docs embedding-eajs" "Modular embedding (EAJS) reference"]]
+ :examples [["./bin/mage docs-generate" "Regenerate every generated doc"]
+ ["./bin/mage docs-generate backend" "Environment variables, config template, API, AI providers, MCP tools, and CLI commands"]
+ ["./bin/mage docs-generate usage-analytics" "Usage analytics reference"]
+ ["./bin/mage docs-generate embedding-sdk" "Embedding SDK reference (builds the SDK package first)"]
+ ["./bin/mage docs-generate embedding-eajs" "Modular embedding (EAJS) reference"]]
:options [["-d" "--dir DIR" "Absolute path of the Metabase checkout to generate the docs in. Defaults to this checkout."]]
:arg-schema [:or
[:tuple]
@@ -435,8 +448,8 @@
"usage-analytics"
"embedding-sdk"
"embedding-eajs"]]]
- :requires [[mage.generate-docs :as generate-docs]]
- :task (task! (generate-docs/run! (first (:arguments parsed)) (:options parsed)))}
+ :requires [[mage.docs-generate :as docs-generate]]
+ :task (task! (docs-generate/run! (first (:arguments parsed)) (:options parsed)))}
analytics:setup
{:doc "Set up usage analytics development mode"
diff --git a/docs/developers-guide/docs.md b/docs/developers-guide/docs.md
index a803a29d2dd..ce1adc0373e 100644
--- a/docs/developers-guide/docs.md
+++ b/docs/developers-guide/docs.md
@@ -28,6 +28,18 @@ bun run lint-docs-links
You can view both commands in the [package.json](https://github.com/metabase/metabase/blob/master/package.json) file under `scripts`.
+## Preview docs as you write them
+
+You'll need [Bun](https://bun.sh) and a checkout of the private `metabase-dot-com` repo at `~/metabase-dot-com`. Then run:
+
+```
+./bin/mage docs-preview
+```
+
+Your browser opens `http://localhost:4000/docs/latest/`, and your edits show up as you save them. Only `/docs/latest/` pages work.
+
+To preview a worktree or another checkout, pass its path. If `metabase-dot-com` is somewhere else, pass `--site DIR` or set `METABASE_DOT_COM_DIR`.
+
## Regenerate docs built from source
Some docs pages are generated from the source code. A scheduled GitHub workflow regenerates them every weekday and opens a PR with any changes.
@@ -41,16 +53,16 @@ bun install
Then run:
```
-./bin/mage generate-docs
+./bin/mage docs-generate
```
To regenerate one group of pages, pass its suite name. For example:
```
-./bin/mage generate-docs backend
+./bin/mage docs-generate backend
```
-To list the suites, run `./bin/mage generate-docs -h`. If any suite fails, the last line of output lists it.
+To list the suites, run `./bin/mage docs-generate -h`. If any suite fails, the last line of output lists it.
## Updating API docs
diff --git a/mage/src/mage/generate_docs.clj b/mage/src/mage/docs_generate.clj
similarity index 93%
rename from mage/src/mage/generate_docs.clj
rename to mage/src/mage/docs_generate.clj
index 76857dfd0db..fd574534ffb 100644
--- a/mage/src/mage/generate_docs.clj
+++ b/mage/src/mage/docs_generate.clj
@@ -1,4 +1,4 @@
-(ns mage.generate-docs
+(ns mage.docs-generate
"Regenerates the docs that are built from source: environment variables, the config template, the API, AI providers,
MCP tools, CLI commands, the usage analytics reference, and the embedding SDK and EAJS references."
(:refer-clojure :exclude [run!])
@@ -13,7 +13,7 @@
;; .github/scripts/check-preresolve-aliases.sh reads the `clojure` alias out of this file.
;; CI runs this list from master on each target branch, so every target branch needs each command.
(def ^:private suites
- "Each suite and its command, in the order the bare `generate-docs` command runs them."
+ "Each suite and its command, in the order the bare `docs-generate` command runs them."
[;; One JVM runs every Clojure generator, so Metabase's namespaces load once rather than once per document.
["backend" ["clojure" "-M:ee:doc" "all-documentation"]]
;; `./bin/mage` installs `bin/bb` in its own checkout only, not in the `--dir` checkout.
@@ -24,7 +24,7 @@
(def ^:private suite->command (into {} suites))
(def suite-names
- "Every suite name, in the order the bare `generate-docs` command runs them."
+ "Every suite name, in the order the bare `docs-generate` command runs them."
(mapv first suites))
(defn- run-suite!
diff --git a/mage/src/mage/docs_preview.clj b/mage/src/mage/docs_preview.clj
new file mode 100644
index 00000000000..b79d9142e47
--- /dev/null
+++ b/mage/src/mage/docs_preview.clj
@@ -0,0 +1,139 @@
+(ns mage.docs-preview
+ "Preview the docs of a Metabase repo in a local metabase-dot-com checkout.
+
+ The docs app of metabase-dot-com reads `METABASE_REPO_PATH` with Vite's `loadEnv`, where the process env wins
+ over `apps/docs/.env`. So we pass the repo in the env, and leave the writer's `.env` as it is."
+ (:require
+ [babashka.fs :as fs]
+ [babashka.http-client :as http]
+ [babashka.process :as p]
+ [clojure.java.browse :as browse]
+ [clojure.string :as str]
+ [mage.color :as c]
+ [mage.util :as u])
+ (:import
+ (java.net InetSocketAddress Socket)))
+
+(set! *warn-on-reflection* true)
+
+(def site-env-var
+ "The env var with the path of the metabase-dot-com checkout."
+ "METABASE_DOT_COM_DIR")
+
+(def ^:private default-site-dir "~/metabase-dot-com")
+
+;; `bun run dev` starts marketing on 4000, and the docs on 4321. Marketing proxies /docs/* to 4321 always, so if an
+;; old docs server holds 4321, the new one moves to 4322 and the proxy shows the old docs with no error.
+(def ^:private site-port 4000)
+(def ^:private docs-port 4321)
+
+(def ^:private preview-url (str "http://localhost:" site-port "/docs/latest/"))
+
+(defn- fail! [message]
+ (u/exit message 1))
+
+(defn- resolve-dir
+ "`dir` as a canonical path. A relative `dir` is relative to `root`, and `~` is the home dir."
+ [root dir]
+ (str (fs/canonicalize (fs/path root (fs/expand-home dir)))))
+
+(defn site-dir
+ "The metabase-dot-com checkout: the `--site` option, then the env var, then ~/metabase-dot-com."
+ [root site-option env]
+ (let [dir (resolve-dir root (or site-option (get env site-env-var) default-site-dir))]
+ (when-not (fs/directory? (fs/path dir "apps" "docs"))
+ (fail! (str "No metabase-dot-com checkout at " (c/yellow dir) ".\n"
+ "Give its path with " (c/green "--site DIR") ", or set " (c/green site-env-var)
+ " in your shell profile.")))
+ dir))
+
+(defn repo-dir
+ "The Metabase repo with the docs: the argument, or `root`."
+ [root repo-arg]
+ (let [dir (resolve-dir root (or repo-arg "."))]
+ (when-not (fs/directory? (fs/path dir "docs"))
+ (fail! (str "No docs/ folder in " (c/yellow dir) ".\n"
+ "Give the path of a Metabase repo or worktree.")))
+ dir))
+
+(defn listening-host
+ "The localhost address where a server listens on `port`, or nil. Astro can listen on IPv4 or IPv6 localhost, so we
+ try both. The IPv6 address is in brackets, so it also goes in a URL."
+ [port]
+ (some (fn [^String host]
+ (try
+ (with-open [socket (Socket.)]
+ (.connect socket (InetSocketAddress. host (int port)) 500)
+ host)
+ (catch Exception _ nil)))
+ ["127.0.0.1" "[::1]"]))
+
+(defn- listening? [port]
+ (some? (listening-host port)))
+
+(defn- stop-hint [site]
+ (str "To stop background servers, run " (c/green "bunx astro dev stop") " in "
+ site "/apps/docs and " site "/apps/marketing."))
+
+(defn- check-ports! [site]
+ (when-let [busy (seq (filter listening? [site-port docs-port]))]
+ (fail! (str "Ports in use: " (str/join ", " busy) ". "
+ "Stop the other dev server (for example, an old docs preview), then try again.\n"
+ (stop-hint site)))))
+
+(defn- check-bun! []
+ (when-not (binding [u/*skip-warning* true] (u/can-run? "bun"))
+ (fail! (str "You don't have " (c/yellow "bun") " installed. Install it from https://bun.sh"))))
+
+(defn- branch [repo]
+ (let [out (str/trim (:out (p/shell {:dir repo :out :string :err :string :continue true}
+ "git" "branch" "--show-current")))]
+ (if (str/blank? out) "(no branch)" out)))
+
+(defn- open-when-ready
+ "Opens the preview in the browser when the dev servers are up. Returns the future, to cancel it."
+ []
+ (future
+ (if (loop [tries 180]
+ (cond
+ (every? listening? [site-port docs-port]) true
+ (pos? tries) (do (Thread/sleep 1000) (recur (dec tries)))
+ :else false))
+ (do
+ ;; The first request renders the page, so the browser does not wait for it. Not to `localhost`: bb resolves
+ ;; it to IPv4, but Node can listen on IPv6 only.
+ (when-let [host (listening-host docs-port)]
+ (try (http/get (str "http://" host ":" docs-port "/docs/latest/") {:throw false :timeout 120000})
+ (catch Exception _ nil)))
+ (try (browse/browse-url preview-url)
+ (catch Exception _ (println "Open" (c/cyan preview-url)))))
+ (println (c/yellow "The dev servers did not start in 3 minutes.")))))
+
+(defn docs-preview!
+ "Runs the metabase-dot-com dev server with the docs of a Metabase repo."
+ [{:keys [arguments options]}]
+ (let [root u/project-root-directory
+ repo (repo-dir root (first arguments))
+ site (site-dir root (:site options) (u/env))]
+ (check-bun!)
+ (check-ports! site)
+ (println (c/magenta "Running: bun install") "in" site)
+ (when-not (zero? (:exit (p/shell {:dir site :continue true} "bun" "install")))
+ (fail! (str (c/yellow "bun install") " failed in " site ".")))
+ (println)
+ (println "Docs: " (c/green repo) (str "(" (branch repo) ")"))
+ (println "Site: " (c/green site))
+ (println "URL: " (c/cyan preview-url))
+ (println "Edits to the docs show in the browser immediately. Only /docs/latest/ pages are served. Ctrl-C stops.")
+ (println)
+ (let [opener (when-not (:no-open options) (open-when-ready))
+ {:keys [exit]} (p/shell {:dir site :extra-env {"METABASE_REPO_PATH" repo} :continue true}
+ "bun" "run" "dev")]
+ ;; When an AI agent runs `astro dev`, Astro starts the server in the background and exits. Ctrl-C stops mage
+ ;; itself, so we get here with the servers up only in that case.
+ (if (and (zero? exit) (listening? docs-port))
+ (do
+ (println (c/yellow "The dev servers run in the background.") (stop-hint site))
+ (some-> opener deref))
+ (some-> opener future-cancel))
+ (u/exit exit))))
diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.clj
index fa8ab7f4e27..87e1804a0e8 100644
--- a/mage/test/mage/core_test.clj
+++ b/mage/test/mage/core_test.clj
@@ -15,9 +15,10 @@
[mage.bot.pr-env-test]
[mage.bot.prompt-test]
[mage.bot.repl-eval-test]
+ [mage.docs-generate-test]
+ [mage.docs-preview-test]
[mage.doctor-test]
[mage.fix-unused-requires-test]
- [mage.generate-docs-test]
[mage.kondo-ratchet-test]
[mage.kondo-ratchets-history-test]
[mage.merge-kondo-ratchets-test]
diff --git a/mage/test/mage/generate_docs_test.clj b/mage/test/mage/docs_generate_test.clj
similarity index 82%
rename from mage/test/mage/generate_docs_test.clj
rename to mage/test/mage/docs_generate_test.clj
index 780f0286a72..558f9cc592c 100644
--- a/mage/test/mage/generate_docs_test.clj
+++ b/mage/test/mage/docs_generate_test.clj
@@ -1,8 +1,8 @@
-(ns mage.generate-docs-test
+(ns mage.docs-generate-test
(:require
[clojure.string :as str]
[clojure.test :refer [deftest is testing]]
- [mage.generate-docs :as generate-docs]
+ [mage.docs-generate :as docs-generate]
[mage.shell :as shell]))
(set! *warn-on-reflection* true)
@@ -22,28 +22,28 @@
{:exit exit, :out [], :err []})))
(defn- run-suites!
- "Run `suite-names` through [[generate-docs/run-suites!]] with the given exit codes.
+ "Run `suite-names` through [[docs-generate/run-suites!]] with the given exit codes.
Returns the failed suites, the recorded commands, and everything printed."
[exits suite-names]
(let [calls (atom [])
out (java.io.StringWriter.)
failed (binding [*out* out]
- (generate-docs/run-suites! (fake-sh calls exits) suite-names))]
+ (docs-generate/run-suites! (fake-sh calls exits) suite-names))]
{:failed failed, :calls @calls, :out (str out)}))
(deftest every-suite-runs-test
- (let [{:keys [failed calls out]} (run-suites! {} generate-docs/suite-names)]
+ (let [{:keys [failed calls out]} (run-suites! {} docs-generate/suite-names)]
(testing "every suite runs once, in order"
- (is (= (map @#'generate-docs/suite->command generate-docs/suite-names) calls)))
+ (is (= (map @#'docs-generate/suite->command docs-generate/suite-names) calls)))
(testing "each suite reports that it finished"
- (doseq [suite generate-docs/suite-names]
+ (doseq [suite docs-generate/suite-names]
(is (str/includes? out (str "Generated " suite " docs")))))
(is (= [] failed))))
(deftest failed-suite-still-runs-later-suites-test
(let [{:keys [failed calls out]} (run-suites! {backend-command 1
["bun" "run" "embedding-sdk:docs:generate"] 1}
- generate-docs/suite-names)]
+ docs-generate/suite-names)]
(testing "every suite runs"
(is (= 4 (count calls))))
(testing "the failed suites are reported, in suite order"
@@ -65,7 +65,7 @@
(let [calls (atom [])]
(with-redefs [shell/sh* (fake-sh calls {backend-command 1})]
(is (= 1 (try
- (with-out-str (generate-docs/run! "backend" {}))
+ (with-out-str (docs-generate/run! "backend" {}))
nil
(catch clojure.lang.ExceptionInfo e
(:babashka/exit (ex-data e)))))))
@@ -73,6 +73,6 @@
(testing "with `:dir`, runs every suite in that checkout"
(let [calls (atom [])]
(with-redefs [shell/sh* (fake-sh calls {})]
- (with-out-str (generate-docs/run! nil {:dir "/tmp/release-checkout"})))
- (is (= (repeat (count generate-docs/suite-names) {:dir "/tmp/release-checkout"})
+ (with-out-str (docs-generate/run! nil {:dir "/tmp/release-checkout"})))
+ (is (= (repeat (count docs-generate/suite-names) {:dir "/tmp/release-checkout"})
(map first @calls))))))
diff --git a/mage/test/mage/docs_preview_test.clj b/mage/test/mage/docs_preview_test.clj
new file mode 100644
index 00000000000..01efa445fde
--- /dev/null
+++ b/mage/test/mage/docs_preview_test.clj
@@ -0,0 +1,68 @@
+(ns mage.docs-preview-test
+ (:require
+ [babashka.fs :as fs]
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing]]
+ [mage.docs-preview :as docs-preview])
+ (:import
+ (java.net InetAddress ServerSocket)))
+
+(set! *warn-on-reflection* true)
+
+(defn- canonical [path]
+ (str (fs/canonicalize path)))
+
+(defn- error-output
+ "What `thunk` prints when it stops with an error."
+ [thunk]
+ (with-out-str (is (thrown? clojure.lang.ExceptionInfo (thunk)))))
+
+(deftest site-dir-test
+ (fs/with-temp-dir [tmp {}]
+ (let [root (canonical tmp)
+ option (str root "/from-option")
+ env (str root "/from-env")]
+ (doseq [dir [option env]]
+ (fs/create-dirs (fs/path dir "apps" "docs")))
+ (testing "the --site option wins over the env var"
+ (is (= option (docs-preview/site-dir root option {docs-preview/site-env-var env}))))
+ (testing "the env var is used when there is no option"
+ (is (= env (docs-preview/site-dir root nil {docs-preview/site-env-var env}))))
+ (testing "a relative path starts at root"
+ (is (= option (docs-preview/site-dir root "from-option" {}))))
+ (testing "a dir without apps/docs is an error that names the option and the env var"
+ (is (re-find #"(?s)No metabase-dot-com checkout.*--site DIR.*METABASE_DOT_COM_DIR"
+ (error-output #(docs-preview/site-dir root (str root "/nope") {}))))))))
+
+(deftest site-dir-default-test
+ (testing "the default is ~/metabase-dot-com"
+ (let [expected (canonical (fs/expand-home "~/metabase-dot-com"))]
+ (if (fs/directory? (fs/path expected "apps" "docs"))
+ (is (= expected (docs-preview/site-dir "/" nil {})))
+ (is (str/includes? (error-output #(docs-preview/site-dir "/" nil {})) expected))))))
+
+(deftest repo-dir-test
+ (fs/with-temp-dir [tmp {}]
+ (let [root (canonical tmp)
+ worktree (str root "/.claude/worktrees/my-feature")]
+ (fs/create-dirs (fs/path root "docs"))
+ (fs/create-dirs (fs/path worktree "docs"))
+ (testing "the default is root"
+ (is (= root (docs-preview/repo-dir root nil))))
+ (testing "a relative path starts at root"
+ (is (= worktree (docs-preview/repo-dir root ".claude/worktrees/my-feature"))))
+ (testing "an absolute path is used as it is"
+ (is (= worktree (docs-preview/repo-dir root worktree))))
+ (testing "a dir without docs/ is an error"
+ (is (re-find #"No docs/ folder" (error-output #(docs-preview/repo-dir root ".claude"))))))))
+
+(deftest listening-host-test
+ ;; Port 0 gives a free port.
+ (doseq [[bind-address expected] [["127.0.0.1" "127.0.0.1"]
+ ["::1" "[::1]"]]]
+ (testing (str "a server on " bind-address " gives " expected ", which goes in a URL")
+ (with-open [server (ServerSocket. 0 1 (InetAddress/getByName bind-address))]
+ (is (= expected (docs-preview/listening-host (.getLocalPort server)))))))
+ (testing "no server gives nil"
+ (let [port (with-open [server (ServerSocket. 0)] (.getLocalPort server))]
+ (is (nil? (docs-preview/listening-host port))))))