Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

docs - analytics events generation (#79377)

Jeff Bruemmer78abe8657405
diff --git a/bin/generate-usage-analytics-docs.bb b/bin/generate-usage-analytics-docs.bb
index d8d63a71911..43a656fc705 100755
--- a/bin/generate-usage-analytics-docs.bb
+++ b/bin/generate-usage-analytics-docs.bb
@@ -19,7 +19,6 @@
{:yaml-dir "resources/instance_analytics/collections/main/usage_analytics"
:template "bin/templates/usage-analytics-reference-intro.md"
:output "docs/usage-and-performance-tools/usage-analytics-reference.md"
- :title "Usage analytics reference"
:sources {:query-sources "src/metabase/lib/schema/info.cljc"
:audit-log-events "src/metabase/audit_app/events/audit_log.clj"
:content-views "resources/migrations/instance_analytics_views/content"
@@ -39,30 +38,37 @@
"Topics filtered out of the v_audit_log SQL view's WHERE clause."
#{"card-read" "card-query" "dashboard-read" "dashboard-query"})
+(def ^:private event-derive-re
+ "Matches `(events/derive! :event/<topic> <parent>)`. Group 1 is the topic; group 2 matches
+ only when the parent is a keyword, so its absence means the call shape drifted and we
+ would silently drop that topic."
+ #"\((?:[a-zA-Z0-9.-]+/)?derive!\s+:event/([a-z0-9-]+)(\s+:)?")
+
(def ^:private content-card-types
"Values from report_card.type yielded by the `type AS entity_type` UNION
arm in the v_content SQL view."
["model" "question"])
(def ^:private view-log-entity-types
- "Entity types written to the view_log table by event handlers in
- src/metabase/view_log/events/view_log.clj. Hardcoded because the
- v_view_log SQL view exposes them via `model AS entity_type` (no SQL
- literals to extract). Update when new model types start being
- view-logged."
- ["card" "collection" "dashboard" "table"])
+ "Entity types written to view_log by the event handlers in
+ src/metabase/view_log/events/view_log.clj and src/metabase/documents/view_log.clj.
+ Hardcoded because `model AS entity_type` leaves no SQL literals to extract; see
+ [[assert-view-log-sql-shape!]]."
+ ["card" "collection" "dashboard" "document" "table"])
;; ---------------------------------------------------------------------------
;; Shared helpers
;; ---------------------------------------------------------------------------
-(defn- non-empty!
- "Returns `coll`, or throws when empty so a moved/renamed source surfaces loudly."
+(defn- sorted-values
+ "Every extractor's output contract: a sorted, deduped, non-empty vector. Throws when empty
+ so a moved or renamed source surfaces loudly instead of silently emptying a documented list."
[label coll]
- (when (empty? coll)
- (throw (ex-info (str "Extracted 0 " label "; source likely moved or changed shape")
- {:extractor label})))
- coll)
+ (let [values (vec (sort (distinct coll)))]
+ (when (empty? values)
+ (throw (ex-info (str "Extracted 0 " label "; source likely moved or changed shape")
+ {:extractor label})))
+ values))
(defn- latest-vN-dir
"Return the directory under `parent` with the highest numeric vN suffix."
@@ -100,8 +106,8 @@
(some-> doc :serdes/meta last :model))
(defn- dashboard? [doc] (= "Dashboard" (serdes-model doc)))
-(defn- card? [doc] (= "Card" (serdes-model doc)))
-(defn- model? [doc] (and (card? doc) (= "model" (some-> doc :type name))))
+(defn- model? [doc] (and (= "Card" (serdes-model doc))
+ (= "model" (some-> doc :type name))))
;; ---------------------------------------------------------------------------
;; Collectors
@@ -116,17 +122,12 @@
(map read-yaml)
(sort-by :path)))
-(defn- dashboard-card-dir
- "Sibling directory next to a dashboard YAML that holds the Card YAMLs
- referenced by the dashboard's inline DashboardCards."
- [dashboard-path]
- (fs/path (fs/parent dashboard-path)
- (str/replace (fs/file-name dashboard-path) #"\.ya?ml$" "")))
-
(defn- dashboard-card-names
- "Return card display names from a dashboard's sibling subdir, ordered by filename."
+ "Card display names from the sibling subdir holding the Card YAMLs that a dashboard's
+ inline DashboardCards reference, ordered by filename."
[dashboard-path]
- (let [dir (dashboard-card-dir dashboard-path)]
+ (let [dir (fs/path (fs/parent dashboard-path)
+ (str/replace (fs/file-name dashboard-path) #"\.ya?ml$" ""))]
(when (fs/directory? dir)
(->> (fs/list-dir dir)
(filter yaml-file?)
@@ -134,24 +135,21 @@
(sort-by :path)
(keep (comp :name :doc))))))
-(defn- model-columns
- "Display names of every column the model exposes."
- [doc]
- (keep :display_name (:result_metadata doc)))
-
(defn- dashboard-entries [yamls]
(for [{:keys [path doc]} yamls
:when (dashboard? doc)]
{:name (:name doc)
:description (:description doc)
- :cards (dashboard-card-names path)}))
+ :label "Cards"
+ :items (dashboard-card-names path)}))
(defn- model-entries [yamls]
(for [{:keys [doc]} yamls
:when (model? doc)]
{:name (:name doc)
:description (:description doc)
- :columns (model-columns doc)}))
+ :label "Columns"
+ :items (keep :display_name (:result_metadata doc))}))
;; ---------------------------------------------------------------------------
;; Categorical-column extractors
@@ -162,34 +160,35 @@
keyword names as sorted strings."
[src]
(let [content (slurp src)
- m (re-find #"(?s)\(mr/def\s+::context.*?\[:enum(.*?)\]\)" content)]
+ ;; Only an optional docstring may sit between `::context` and its `[:enum`, so a
+ ;; `::context` that stops being an enum throws below rather than matching some later
+ ;; definition's `[:enum` and yielding a plausible-looking wrong list.
+ m (re-find #"(?s)\(mr/def\s+::context\s*(?:\"[^\"]*\"\s*)?\[:enum(.*?)\]\)" content)]
(when-not m
(throw (ex-info (str "Could not find (mr/def ::context [:enum ...]) in " src)
{:source src})))
(->> (re-seq #":([a-z][a-z0-9-]*(?:/[a-z][a-z0-9-]*)?)" (second m))
(map second)
(remove #(str/includes? % "/"))
- distinct
- sort
- vec
- (non-empty! "query sources"))))
+ (sorted-values "query sources"))))
(defn- audit-log-topics
- "Grep `(derive :event/<topic> ::<parent>)` from the audit-log event file,
- then apply the v_audit_log SQL view's rename and exclusion rules so the
- list matches what users see in the analytics model."
+ "Grep [[event-derive-re]] over the audit-log event file, then apply the v_audit_log SQL
+ view's rename and exclusion rules so the list matches what users see in the analytics
+ model."
[src]
- (let [content (slurp src)
- topics (->> (re-seq #"\(derive\s+:event/([a-z0-9-]+)\s+::[a-z0-9-]+\)" content)
- (map second)
- distinct)]
- (->> topics
+ (let [forms (re-seq event-derive-re (slurp src))
+ ;; Guard against partial drift: `sorted-values` only catches the case where every
+ ;; call site changed shape, not where a few did and get silently dropped.
+ drifted (keep #(when-not (nth % 2) (second %)) forms)]
+ (when (seq drifted)
+ (throw (ex-info (str "`derive!` parent is no longer a keyword for: " (str/join ", " drifted))
+ {:source src :topics (vec drifted)})))
+ (->> forms
+ (map second)
(remove audit-topic-exclusions)
(map #(get audit-topic-renames % %))
- distinct
- sort
- vec
- (non-empty! "audit log topics"))))
+ (sorted-values "audit log topics"))))
(defn- content-entity-types
"Pull `'X' AS entity_type` literals from the latest mysql-content.sql view,
@@ -203,10 +202,7 @@
has-card-arm? (and (re-find #"(?i)type\s+as\s+entity_type" content)
(re-find #"(?i)report_card" content))]
(->> (concat literals (when has-card-arm? content-card-types))
- distinct
- sort
- vec
- (non-empty! "content entity types"))))
+ (sorted-values "content entity types"))))
(defn- assert-view-log-sql-shape!
"Throws if the v_view_log SQL view stops using `model AS entity_type` —
@@ -217,26 +213,21 @@
(throw (ex-info "v_view_log SQL no longer uses `model AS entity_type`. Re-derive view-log-entity-types manually."
{:source sql-file})))))
-(defn- categorical-sections [{:keys [sources]}]
+(defn- categorical-entries
+ "The fixed-value column lists, each read from its canonical source."
+ [sources]
[{:name "Activity log topics"
:description "The Topic column on the [Activity log](#activity-log) model takes one of:"
- :values-fn #(audit-log-topics (:audit-log-events sources))}
+ :items (audit-log-topics (:audit-log-events sources))}
{:name "Query log query sources"
:description "The Query Source column on the [Query log](#query-log) model takes one of:"
- :values-fn #(query-sources (:query-sources sources))}
+ :items (query-sources (:query-sources sources))}
{:name "Content entity types"
:description "The Entity Type column on the [Content](#content) model takes one of:"
- :values-fn #(content-entity-types (:content-views sources))}
+ :items (content-entity-types (:content-views sources))}
{:name "View log entity types"
:description "The Entity Type column on the [View log](#view-log) model takes one of:"
- :values-fn (constantly view-log-entity-types)}])
-
-(defn- categorical-entries [sections]
- (mapv (fn [{:keys [name description values-fn]}]
- {:name name
- :description description
- :values (values-fn)})
- sections))
+ :items view-log-entity-types}])
;; ---------------------------------------------------------------------------
;; Markdown rendering
@@ -245,59 +236,19 @@
(defn- bullet-list [items]
(str/join "\n" (map #(str "- " %) items)))
-(defn- section-header [level text]
- (str (apply str (repeat level "#")) " " text))
-
-(defn- labelled-list-renderer
- "Returns a fn that renders items as `<label>:\\n\\n<bullets>`, falling back
- to `empty-label` when the seq is empty."
- [label empty-label]
- (fn [items]
- (if (seq items)
- (str label ":\n\n" (bullet-list items))
- empty-label)))
-
-(defn- bare-list
- "Render `items` as a plain bullet list, or `nil` when empty."
- [items]
- (when (seq items)
- (bullet-list items)))
-
-(defn- entity-markdown
- "Render one `###` subsection from `{:name :description :items :render-items}`."
- [{:keys [name description items render-items]}]
- (str/join "\n\n"
- (remove str/blank?
- [(section-header 3 name)
- description
- (render-items items)])))
-
-(defn- entities-markdown
- [{:keys [items-key render-items]} entities]
- (->> entities
- (map (fn [e]
- (entity-markdown {:name (:name e)
- :description (:description e)
- :items (items-key e)
- :render-items render-items})))
+(defn- section-markdown
+ "Render one `###` subsection from `{:name :description :label :items}`. The list is
+ dropped when `items` is empty, and rendered bare when there is no `label`."
+ [{:keys [name description label items]}]
+ (->> [(str "### " name)
+ description
+ (when (seq items)
+ (str (when label (str label ":\n\n")) (bullet-list items)))]
+ (remove str/blank?)
(str/join "\n\n")))
-(def ^:private dashboard-section
- {:items-key :cards
- :render-items (labelled-list-renderer "Cards" "_No cards found._")})
-
-(def ^:private model-section
- {:items-key :columns
- :render-items (labelled-list-renderer "Columns" "_No columns found._")})
-
-(def ^:private categorical-section
- {:items-key :values
- :render-items bare-list})
-
-(defn- intro-markdown
- "Substitute `{{title}}` into `template-content`."
- [template-content title]
- (str/replace template-content "{{title}}" title))
+(defn- sections-markdown [entries]
+ (str/join "\n\n" (map section-markdown entries)))
(defn- document-markdown
[{:keys [intro dashboards models categorical]}]
@@ -305,19 +256,19 @@
(remove str/blank?
[(str/trimr intro)
"## Dashboards"
- (entities-markdown dashboard-section dashboards)
+ (sections-markdown dashboards)
"## Models"
- (entities-markdown model-section models)
+ (sections-markdown models)
"## Categorical column values"
"Some columns in the models above hold one of a fixed set of values."
- (entities-markdown categorical-section categorical)]))
+ (sections-markdown categorical)]))
"\n"))
;; ---------------------------------------------------------------------------
;; Entry point
;; ---------------------------------------------------------------------------
-(defn- generate! [{:keys [yaml-dir template output title sources] :as cfg}]
+(defn- generate! [{:keys [yaml-dir template output sources]}]
(when-not (fs/directory? yaml-dir)
(throw (ex-info (str "YAML directory does not exist: " yaml-dir)
{:yaml-dir yaml-dir
@@ -326,10 +277,9 @@
(let [yamls (top-level-yamls yaml-dir)
dashboards (dashboard-entries yamls)
models (model-entries yamls)
- categorical (categorical-entries (categorical-sections cfg))
- intro (intro-markdown (slurp template) title)
+ categorical (categorical-entries sources)
content (document-markdown
- {:intro intro
+ {:intro (slurp template)
:dashboards dashboards
:models models
:categorical categorical})]
@@ -340,11 +290,7 @@
(count dashboards)
(count models)
(count categorical)
- (str/join ", " (map (comp count :values) categorical))))
- {:dashboards (count dashboards)
- :models (count models)
- :categorical (count categorical)
- :output output}))
+ (str/join ", " (map (comp count :items) categorical))))))
(defn -main [& _args]
(try
diff --git a/bin/templates/usage-analytics-reference-intro.md b/bin/templates/usage-analytics-reference-intro.md
index 8c032abf82c..bac0cbad1f8 100644
--- a/bin/templates/usage-analytics-reference-intro.md
+++ b/bin/templates/usage-analytics-reference-intro.md
@@ -1,9 +1,9 @@
---
-title: {{title}}
+title: Usage analytics reference
summary: "A list of the dashboards and models included in the Usage Analytics collection."
---
-# {{title}}
+# Usage analytics reference
_This page is generated by `bin/generate-usage-analytics-docs.bb`._