The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
docs - add mage docs-preview (#85425)
Jeff Bruemmerdbd86b4960ec
diff --git a/.github/scripts/check-preresolve-aliases.sh b/.github/scripts/check-preresolve-aliases.shindex 8487626abb7..44984dc8960 100755--- a/.github/scripts/check-preresolve-aliases.sh+++ b/.github/scripts/check-preresolve-aliases.sh@@ -81,11 +81,11 @@ require_scan() { # require_scan <what> <result>workflow_checks=$( { scan ee; scan oss; } || true)clojure_checks=$(scan_project_tests run-clojure-checks! . || true)migration_checks=$(scan_project_tests run-migration-checks! bin/lint-migrations-file || true)-docs_checks=$(mage_aliases . < mage/src/mage/generate_docs.clj || true)+docs_checks=$(mage_aliases . < mage/src/mage/docs_generate.clj || true)require_scan ".github/" "$workflow_checks"require_scan "run-clojure-checks! (mage/src/mage/project_tests.clj)" "$clojure_checks"require_scan "run-migration-checks! (mage/src/mage/project_tests.clj)" "$migration_checks"-require_scan "mage/src/mage/generate_docs.clj" "$docs_checks"+require_scan "mage/src/mage/docs_generate.clj" "$docs_checks"scanned="$(printf '%s\n' "$workflow_checks" "$clojure_checks" "$migration_checks" "$docs_checks" | sort -u)"diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.ymlindex fefca46fd4f..7e22320473f 100644--- a/.github/workflows/cache-warm.yml+++ b/.github/workflows/cache-warm.yml@@ -128,7 +128,7 @@ jobs:":dev:drivers:build:build-dev:build-test:ci" # build-scripts.yml":test" # build-scripts.yml (bin/build unit tests)":ee:drivers:load-namespaces" # backend.yml (load-namespaces)- ":ee:doc" # mage generate-docs (docs-generate-base.yml)+ ":ee:doc" # mage docs-generate (docs-generate-base.yml)":kondo" # resolve-backport-conflicts.yml (replace-deps)":run:ee" # cross-branch-migration-test":dev:ee:migrate" # cross-branch-migration-testdiff --git a/.github/workflows/docs-generate-base.yml b/.github/workflows/docs-generate-base.ymlindex 96956c18c88..622dd865693 100644--- a/.github/workflows/docs-generate-base.yml+++ b/.github/workflows/docs-generate-base.yml@@ -50,7 +50,7 @@ jobs:uses: ./.github/actions/prepare-frontend- name: Generate documentation- run: docs-generator/bin/mage generate-docs --dir "$GITHUB_WORKSPACE"+ run: docs-generator/bin/mage docs-generate --dir "$GITHUB_WORKSPACE"- name: Check for changesid: check_changes@@ -117,7 +117,7 @@ jobs:$([ "$TOTAL_CHANGES" -gt 10 ] && echo "- ... and $((TOTAL_CHANGES - 10)) more files")## Generation command- Documentation was generated using \`./bin/mage generate-docs\`.+ Documentation was generated using \`./bin/mage docs-generate\`.Please review the changes and merge if they look correct." \--label "Type:Documentation" \diff --git a/bb.edn b/bb.ednindex fd6757cdb16..c30ea1d4c2d 100644--- a/bb.edn+++ b/bb.edn@@ -122,6 +122,19 @@:requires [[mage.start-maildev :as start-maildev]]:task (task! (start-maildev/start-maildev!))}+ docs-preview+ {:doc "Preview the docs of a Metabase repo in your metabase-dot-com checkout, at http://localhost:4000/docs/latest/"+ :examples [["./bin/mage docs-preview" "Preview the docs of this repo"]+ ["./bin/mage docs-preview .claude/worktrees/my-feature" "Preview the docs of a worktree (a relative path starts at this repo)"]+ ["./bin/mage docs-preview ~/code/other-metabase" "Preview the docs of another checkout"]+ ["./bin/mage docs-preview --site ~/code/www" "Use the metabase-dot-com checkout at ~/code/www"]+ ["METABASE_DOT_COM_DIR=~/code/www ./bin/mage docs-preview" "The same, from the env. Set it in your shell profile to use it always"]]+ :requires [[mage.docs-preview :as docs-preview]]+ :options [[nil "--site DIR" "The metabase-dot-com checkout. A relative path starts at this repo. Default: $METABASE_DOT_COM_DIR, then ~/metabase-dot-com"]+ [nil "--no-open" "Do not open the browser"]]+ :arg-schema [:or [:tuple] [:tuple [:string {:name "repo-path" :description "The Metabase repo or worktree with the docs. Default: this repo"}]]]+ :task (task! (docs-preview/docs-preview! parsed))}+tail-d{:doc "Tail the #d debug log in real time (see dev.debug). Asks the running REPL for the exact path; falls back to DEV_DEBUG_LOG or /tmp/<dir>_debug.log.":examples [["./bin/mage tail-d" "tail the #d log, with bat syntax highlighting if available"]]@@ -420,13 +433,13 @@:requires [[mage.cursive-resolve-as :as cursive-resolve-as]]:task (task! (cursive-resolve-as/do-it))}- generate-docs+ docs-generate{:doc "Regenerate the docs that are built from source. The backend suite runs every Clojure generator in one JVM."- :examples [["./bin/mage generate-docs" "Regenerate every generated doc"]- ["./bin/mage generate-docs backend" "Environment variables, config template, API, AI providers, MCP tools, and CLI commands"]- ["./bin/mage generate-docs usage-analytics" "Usage analytics reference"]- ["./bin/mage generate-docs embedding-sdk" "Embedding SDK reference (builds the SDK package first)"]- ["./bin/mage generate-docs embedding-eajs" "Modular embedding (EAJS) reference"]]+ :examples [["./bin/mage docs-generate" "Regenerate every generated doc"]+ ["./bin/mage docs-generate backend" "Environment variables, config template, API, AI providers, MCP tools, and CLI commands"]+ ["./bin/mage docs-generate usage-analytics" "Usage analytics reference"]+ ["./bin/mage docs-generate embedding-sdk" "Embedding SDK reference (builds the SDK package first)"]+ ["./bin/mage docs-generate embedding-eajs" "Modular embedding (EAJS) reference"]]:options [["-d" "--dir DIR" "Absolute path of the Metabase checkout to generate the docs in. Defaults to this checkout."]]:arg-schema [:or[:tuple]@@ -435,8 +448,8 @@"usage-analytics""embedding-sdk""embedding-eajs"]]]- :requires [[mage.generate-docs :as generate-docs]]- :task (task! (generate-docs/run! (first (:arguments parsed)) (:options parsed)))}+ :requires [[mage.docs-generate :as docs-generate]]+ :task (task! (docs-generate/run! (first (:arguments parsed)) (:options parsed)))}analytics:setup{:doc "Set up usage analytics development mode"diff --git a/docs/developers-guide/docs.md b/docs/developers-guide/docs.mdindex a803a29d2dd..ce1adc0373e 100644--- a/docs/developers-guide/docs.md+++ b/docs/developers-guide/docs.md@@ -28,6 +28,18 @@ bun run lint-docs-linksYou can view both commands in the [package.json](https://github.com/metabase/metabase/blob/master/package.json) file under `scripts`.+## Preview docs as you write them++You'll need [Bun](https://bun.sh) and a checkout of the private `metabase-dot-com` repo at `~/metabase-dot-com`. Then run:++```+./bin/mage docs-preview+```++Your browser opens `http://localhost:4000/docs/latest/`, and your edits show up as you save them. Only `/docs/latest/` pages work.++To preview a worktree or another checkout, pass its path. If `metabase-dot-com` is somewhere else, pass `--site DIR` or set `METABASE_DOT_COM_DIR`.+## Regenerate docs built from sourceSome docs pages are generated from the source code. A scheduled GitHub workflow regenerates them every weekday and opens a PR with any changes.@@ -41,16 +53,16 @@ bun installThen run:```-./bin/mage generate-docs+./bin/mage docs-generate```To regenerate one group of pages, pass its suite name. For example:```-./bin/mage generate-docs backend+./bin/mage docs-generate backend```-To list the suites, run `./bin/mage generate-docs -h`. If any suite fails, the last line of output lists it.+To list the suites, run `./bin/mage docs-generate -h`. If any suite fails, the last line of output lists it.## Updating API docsdiff --git a/mage/src/mage/generate_docs.clj b/mage/src/mage/docs_generate.cljsimilarity index 93%rename from mage/src/mage/generate_docs.cljrename to mage/src/mage/docs_generate.cljindex 76857dfd0db..fd574534ffb 100644--- a/mage/src/mage/generate_docs.clj+++ b/mage/src/mage/docs_generate.clj@@ -1,4 +1,4 @@-(ns mage.generate-docs+(ns mage.docs-generate"Regenerates the docs that are built from source: environment variables, the config template, the API, AI providers,MCP tools, CLI commands, the usage analytics reference, and the embedding SDK and EAJS references."(:refer-clojure :exclude [run!])@@ -13,7 +13,7 @@;; .github/scripts/check-preresolve-aliases.sh reads the `clojure` alias out of this file.;; CI runs this list from master on each target branch, so every target branch needs each command.(def ^:private suites- "Each suite and its command, in the order the bare `generate-docs` command runs them."+ "Each suite and its command, in the order the bare `docs-generate` command runs them."[;; One JVM runs every Clojure generator, so Metabase's namespaces load once rather than once per document.["backend" ["clojure" "-M:ee:doc" "all-documentation"]];; `./bin/mage` installs `bin/bb` in its own checkout only, not in the `--dir` checkout.@@ -24,7 +24,7 @@(def ^:private suite->command (into {} suites))(def suite-names- "Every suite name, in the order the bare `generate-docs` command runs them."+ "Every suite name, in the order the bare `docs-generate` command runs them."(mapv first suites))(defn- run-suite!diff --git a/mage/src/mage/docs_preview.clj b/mage/src/mage/docs_preview.cljnew file mode 100644index 00000000000..b79d9142e47--- /dev/null+++ b/mage/src/mage/docs_preview.clj@@ -0,0 +1,139 @@+(ns mage.docs-preview+ "Preview the docs of a Metabase repo in a local metabase-dot-com checkout.++ The docs app of metabase-dot-com reads `METABASE_REPO_PATH` with Vite's `loadEnv`, where the process env wins+ over `apps/docs/.env`. So we pass the repo in the env, and leave the writer's `.env` as it is."+ (:require+ [babashka.fs :as fs]+ [babashka.http-client :as http]+ [babashka.process :as p]+ [clojure.java.browse :as browse]+ [clojure.string :as str]+ [mage.color :as c]+ [mage.util :as u])+ (:import+ (java.net InetSocketAddress Socket)))++(set! *warn-on-reflection* true)++(def site-env-var+ "The env var with the path of the metabase-dot-com checkout."+ "METABASE_DOT_COM_DIR")++(def ^:private default-site-dir "~/metabase-dot-com")++;; `bun run dev` starts marketing on 4000, and the docs on 4321. Marketing proxies /docs/* to 4321 always, so if an+;; old docs server holds 4321, the new one moves to 4322 and the proxy shows the old docs with no error.+(def ^:private site-port 4000)+(def ^:private docs-port 4321)++(def ^:private preview-url (str "http://localhost:" site-port "/docs/latest/"))++(defn- fail! [message]+ (u/exit message 1))++(defn- resolve-dir+ "`dir` as a canonical path. A relative `dir` is relative to `root`, and `~` is the home dir."+ [root dir]+ (str (fs/canonicalize (fs/path root (fs/expand-home dir)))))++(defn site-dir+ "The metabase-dot-com checkout: the `--site` option, then the env var, then ~/metabase-dot-com."+ [root site-option env]+ (let [dir (resolve-dir root (or site-option (get env site-env-var) default-site-dir))]+ (when-not (fs/directory? (fs/path dir "apps" "docs"))+ (fail! (str "No metabase-dot-com checkout at " (c/yellow dir) ".\n"+ "Give its path with " (c/green "--site DIR") ", or set " (c/green site-env-var)+ " in your shell profile.")))+ dir))++(defn repo-dir+ "The Metabase repo with the docs: the argument, or `root`."+ [root repo-arg]+ (let [dir (resolve-dir root (or repo-arg "."))]+ (when-not (fs/directory? (fs/path dir "docs"))+ (fail! (str "No docs/ folder in " (c/yellow dir) ".\n"+ "Give the path of a Metabase repo or worktree.")))+ dir))++(defn listening-host+ "The localhost address where a server listens on `port`, or nil. Astro can listen on IPv4 or IPv6 localhost, so we+ try both. The IPv6 address is in brackets, so it also goes in a URL."+ [port]+ (some (fn [^String host]+ (try+ (with-open [socket (Socket.)]+ (.connect socket (InetSocketAddress. host (int port)) 500)+ host)+ (catch Exception _ nil)))+ ["127.0.0.1" "[::1]"]))++(defn- listening? [port]+ (some? (listening-host port)))++(defn- stop-hint [site]+ (str "To stop background servers, run " (c/green "bunx astro dev stop") " in "+ site "/apps/docs and " site "/apps/marketing."))++(defn- check-ports! [site]+ (when-let [busy (seq (filter listening? [site-port docs-port]))]+ (fail! (str "Ports in use: " (str/join ", " busy) ". "+ "Stop the other dev server (for example, an old docs preview), then try again.\n"+ (stop-hint site)))))++(defn- check-bun! []+ (when-not (binding [u/*skip-warning* true] (u/can-run? "bun"))+ (fail! (str "You don't have " (c/yellow "bun") " installed. Install it from https://bun.sh"))))++(defn- branch [repo]+ (let [out (str/trim (:out (p/shell {:dir repo :out :string :err :string :continue true}+ "git" "branch" "--show-current")))]+ (if (str/blank? out) "(no branch)" out)))++(defn- open-when-ready+ "Opens the preview in the browser when the dev servers are up. Returns the future, to cancel it."+ []+ (future+ (if (loop [tries 180]+ (cond+ (every? listening? [site-port docs-port]) true+ (pos? tries) (do (Thread/sleep 1000) (recur (dec tries)))+ :else false))+ (do+ ;; The first request renders the page, so the browser does not wait for it. Not to `localhost`: bb resolves+ ;; it to IPv4, but Node can listen on IPv6 only.+ (when-let [host (listening-host docs-port)]+ (try (http/get (str "http://" host ":" docs-port "/docs/latest/") {:throw false :timeout 120000})+ (catch Exception _ nil)))+ (try (browse/browse-url preview-url)+ (catch Exception _ (println "Open" (c/cyan preview-url)))))+ (println (c/yellow "The dev servers did not start in 3 minutes.")))))++(defn docs-preview!+ "Runs the metabase-dot-com dev server with the docs of a Metabase repo."+ [{:keys [arguments options]}]+ (let [root u/project-root-directory+ repo (repo-dir root (first arguments))+ site (site-dir root (:site options) (u/env))]+ (check-bun!)+ (check-ports! site)+ (println (c/magenta "Running: bun install") "in" site)+ (when-not (zero? (:exit (p/shell {:dir site :continue true} "bun" "install")))+ (fail! (str (c/yellow "bun install") " failed in " site ".")))+ (println)+ (println "Docs: " (c/green repo) (str "(" (branch repo) ")"))+ (println "Site: " (c/green site))+ (println "URL: " (c/cyan preview-url))+ (println "Edits to the docs show in the browser immediately. Only /docs/latest/ pages are served. Ctrl-C stops.")+ (println)+ (let [opener (when-not (:no-open options) (open-when-ready))+ {:keys [exit]} (p/shell {:dir site :extra-env {"METABASE_REPO_PATH" repo} :continue true}+ "bun" "run" "dev")]+ ;; When an AI agent runs `astro dev`, Astro starts the server in the background and exits. Ctrl-C stops mage+ ;; itself, so we get here with the servers up only in that case.+ (if (and (zero? exit) (listening? docs-port))+ (do+ (println (c/yellow "The dev servers run in the background.") (stop-hint site))+ (some-> opener deref))+ (some-> opener future-cancel))+ (u/exit exit))))diff --git a/mage/test/mage/core_test.clj b/mage/test/mage/core_test.cljindex fa8ab7f4e27..87e1804a0e8 100644--- a/mage/test/mage/core_test.clj+++ b/mage/test/mage/core_test.clj@@ -15,9 +15,10 @@[mage.bot.pr-env-test][mage.bot.prompt-test][mage.bot.repl-eval-test]+ [mage.docs-generate-test]+ [mage.docs-preview-test][mage.doctor-test][mage.fix-unused-requires-test]- [mage.generate-docs-test][mage.kondo-ratchet-test][mage.kondo-ratchets-history-test][mage.merge-kondo-ratchets-test]diff --git a/mage/test/mage/generate_docs_test.clj b/mage/test/mage/docs_generate_test.cljsimilarity index 82%rename from mage/test/mage/generate_docs_test.cljrename to mage/test/mage/docs_generate_test.cljindex 780f0286a72..558f9cc592c 100644--- a/mage/test/mage/generate_docs_test.clj+++ b/mage/test/mage/docs_generate_test.clj@@ -1,8 +1,8 @@-(ns mage.generate-docs-test+(ns mage.docs-generate-test(:require[clojure.string :as str][clojure.test :refer [deftest is testing]]- [mage.generate-docs :as generate-docs]+ [mage.docs-generate :as docs-generate][mage.shell :as shell]))(set! *warn-on-reflection* true)@@ -22,28 +22,28 @@{:exit exit, :out [], :err []})))(defn- run-suites!- "Run `suite-names` through [[generate-docs/run-suites!]] with the given exit codes.+ "Run `suite-names` through [[docs-generate/run-suites!]] with the given exit codes.Returns the failed suites, the recorded commands, and everything printed."[exits suite-names](let [calls (atom [])out (java.io.StringWriter.)failed (binding [*out* out]- (generate-docs/run-suites! (fake-sh calls exits) suite-names))]+ (docs-generate/run-suites! (fake-sh calls exits) suite-names))]{:failed failed, :calls @calls, :out (str out)}))(deftest every-suite-runs-test- (let [{:keys [failed calls out]} (run-suites! {} generate-docs/suite-names)]+ (let [{:keys [failed calls out]} (run-suites! {} docs-generate/suite-names)](testing "every suite runs once, in order"- (is (= (map @#'generate-docs/suite->command generate-docs/suite-names) calls)))+ (is (= (map @#'docs-generate/suite->command docs-generate/suite-names) calls)))(testing "each suite reports that it finished"- (doseq [suite generate-docs/suite-names]+ (doseq [suite docs-generate/suite-names](is (str/includes? out (str "Generated " suite " docs")))))(is (= [] failed))))(deftest failed-suite-still-runs-later-suites-test(let [{:keys [failed calls out]} (run-suites! {backend-command 1["bun" "run" "embedding-sdk:docs:generate"] 1}- generate-docs/suite-names)]+ docs-generate/suite-names)](testing "every suite runs"(is (= 4 (count calls))))(testing "the failed suites are reported, in suite order"@@ -65,7 +65,7 @@(let [calls (atom [])](with-redefs [shell/sh* (fake-sh calls {backend-command 1})](is (= 1 (try- (with-out-str (generate-docs/run! "backend" {}))+ (with-out-str (docs-generate/run! "backend" {}))nil(catch clojure.lang.ExceptionInfo e(:babashka/exit (ex-data e)))))))@@ -73,6 +73,6 @@(testing "with `:dir`, runs every suite in that checkout"(let [calls (atom [])](with-redefs [shell/sh* (fake-sh calls {})]- (with-out-str (generate-docs/run! nil {:dir "/tmp/release-checkout"})))- (is (= (repeat (count generate-docs/suite-names) {:dir "/tmp/release-checkout"})+ (with-out-str (docs-generate/run! nil {:dir "/tmp/release-checkout"})))+ (is (= (repeat (count docs-generate/suite-names) {:dir "/tmp/release-checkout"})(map first @calls))))))diff --git a/mage/test/mage/docs_preview_test.clj b/mage/test/mage/docs_preview_test.cljnew file mode 100644index 00000000000..01efa445fde--- /dev/null+++ b/mage/test/mage/docs_preview_test.clj@@ -0,0 +1,68 @@+(ns mage.docs-preview-test+ (:require+ [babashka.fs :as fs]+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing]]+ [mage.docs-preview :as docs-preview])+ (:import+ (java.net InetAddress ServerSocket)))++(set! *warn-on-reflection* true)++(defn- canonical [path]+ (str (fs/canonicalize path)))++(defn- error-output+ "What `thunk` prints when it stops with an error."+ [thunk]+ (with-out-str (is (thrown? clojure.lang.ExceptionInfo (thunk)))))++(deftest site-dir-test+ (fs/with-temp-dir [tmp {}]+ (let [root (canonical tmp)+ option (str root "/from-option")+ env (str root "/from-env")]+ (doseq [dir [option env]]+ (fs/create-dirs (fs/path dir "apps" "docs")))+ (testing "the --site option wins over the env var"+ (is (= option (docs-preview/site-dir root option {docs-preview/site-env-var env}))))+ (testing "the env var is used when there is no option"+ (is (= env (docs-preview/site-dir root nil {docs-preview/site-env-var env}))))+ (testing "a relative path starts at root"+ (is (= option (docs-preview/site-dir root "from-option" {}))))+ (testing "a dir without apps/docs is an error that names the option and the env var"+ (is (re-find #"(?s)No metabase-dot-com checkout.*--site DIR.*METABASE_DOT_COM_DIR"+ (error-output #(docs-preview/site-dir root (str root "/nope") {}))))))))++(deftest site-dir-default-test+ (testing "the default is ~/metabase-dot-com"+ (let [expected (canonical (fs/expand-home "~/metabase-dot-com"))]+ (if (fs/directory? (fs/path expected "apps" "docs"))+ (is (= expected (docs-preview/site-dir "/" nil {})))+ (is (str/includes? (error-output #(docs-preview/site-dir "/" nil {})) expected))))))++(deftest repo-dir-test+ (fs/with-temp-dir [tmp {}]+ (let [root (canonical tmp)+ worktree (str root "/.claude/worktrees/my-feature")]+ (fs/create-dirs (fs/path root "docs"))+ (fs/create-dirs (fs/path worktree "docs"))+ (testing "the default is root"+ (is (= root (docs-preview/repo-dir root nil))))+ (testing "a relative path starts at root"+ (is (= worktree (docs-preview/repo-dir root ".claude/worktrees/my-feature"))))+ (testing "an absolute path is used as it is"+ (is (= worktree (docs-preview/repo-dir root worktree))))+ (testing "a dir without docs/ is an error"+ (is (re-find #"No docs/ folder" (error-output #(docs-preview/repo-dir root ".claude"))))))))++(deftest listening-host-test+ ;; Port 0 gives a free port.+ (doseq [[bind-address expected] [["127.0.0.1" "127.0.0.1"]+ ["::1" "[::1]"]]]+ (testing (str "a server on " bind-address " gives " expected ", which goes in a URL")+ (with-open [server (ServerSocket. 0 1 (InetAddress/getByName bind-address))]+ (is (= expected (docs-preview/listening-host (.getLocalPort server)))))))+ (testing "no server gives nil"+ (let [port (with-open [server (ServerSocket. 0)] (.getLocalPort server))]+ (is (nil? (docs-preview/listening-host port))))))