The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
docs - analytics events generation (#79377)
Jeff Bruemmer78abe8657405
diff --git a/bin/generate-usage-analytics-docs.bb b/bin/generate-usage-analytics-docs.bbindex d8d63a71911..43a656fc705 100755--- a/bin/generate-usage-analytics-docs.bb+++ b/bin/generate-usage-analytics-docs.bb@@ -19,7 +19,6 @@{:yaml-dir "resources/instance_analytics/collections/main/usage_analytics":template "bin/templates/usage-analytics-reference-intro.md":output "docs/usage-and-performance-tools/usage-analytics-reference.md"- :title "Usage analytics reference":sources {:query-sources "src/metabase/lib/schema/info.cljc":audit-log-events "src/metabase/audit_app/events/audit_log.clj":content-views "resources/migrations/instance_analytics_views/content"@@ -39,30 +38,37 @@"Topics filtered out of the v_audit_log SQL view's WHERE clause."#{"card-read" "card-query" "dashboard-read" "dashboard-query"})+(def ^:private event-derive-re+ "Matches `(events/derive! :event/<topic> <parent>)`. Group 1 is the topic; group 2 matches+ only when the parent is a keyword, so its absence means the call shape drifted and we+ would silently drop that topic."+ #"\((?:[a-zA-Z0-9.-]+/)?derive!\s+:event/([a-z0-9-]+)(\s+:)?")+(def ^:private content-card-types"Values from report_card.type yielded by the `type AS entity_type` UNIONarm in the v_content SQL view."["model" "question"])(def ^:private view-log-entity-types- "Entity types written to the view_log table by event handlers in- src/metabase/view_log/events/view_log.clj. Hardcoded because the- v_view_log SQL view exposes them via `model AS entity_type` (no SQL- literals to extract). Update when new model types start being- view-logged."- ["card" "collection" "dashboard" "table"])+ "Entity types written to view_log by the event handlers in+ src/metabase/view_log/events/view_log.clj and src/metabase/documents/view_log.clj.+ Hardcoded because `model AS entity_type` leaves no SQL literals to extract; see+ [[assert-view-log-sql-shape!]]."+ ["card" "collection" "dashboard" "document" "table"]);; ---------------------------------------------------------------------------;; Shared helpers;; ----------------------------------------------------------------------------(defn- non-empty!- "Returns `coll`, or throws when empty so a moved/renamed source surfaces loudly."+(defn- sorted-values+ "Every extractor's output contract: a sorted, deduped, non-empty vector. Throws when empty+ so a moved or renamed source surfaces loudly instead of silently emptying a documented list."[label coll]- (when (empty? coll)- (throw (ex-info (str "Extracted 0 " label "; source likely moved or changed shape")- {:extractor label})))- coll)+ (let [values (vec (sort (distinct coll)))]+ (when (empty? values)+ (throw (ex-info (str "Extracted 0 " label "; source likely moved or changed shape")+ {:extractor label})))+ values))(defn- latest-vN-dir"Return the directory under `parent` with the highest numeric vN suffix."@@ -100,8 +106,8 @@(some-> doc :serdes/meta last :model))(defn- dashboard? [doc] (= "Dashboard" (serdes-model doc)))-(defn- card? [doc] (= "Card" (serdes-model doc)))-(defn- model? [doc] (and (card? doc) (= "model" (some-> doc :type name))))+(defn- model? [doc] (and (= "Card" (serdes-model doc))+ (= "model" (some-> doc :type name))));; ---------------------------------------------------------------------------;; Collectors@@ -116,17 +122,12 @@(map read-yaml)(sort-by :path)))-(defn- dashboard-card-dir- "Sibling directory next to a dashboard YAML that holds the Card YAMLs- referenced by the dashboard's inline DashboardCards."- [dashboard-path]- (fs/path (fs/parent dashboard-path)- (str/replace (fs/file-name dashboard-path) #"\.ya?ml$" "")))-(defn- dashboard-card-names- "Return card display names from a dashboard's sibling subdir, ordered by filename."+ "Card display names from the sibling subdir holding the Card YAMLs that a dashboard's+ inline DashboardCards reference, ordered by filename."[dashboard-path]- (let [dir (dashboard-card-dir dashboard-path)]+ (let [dir (fs/path (fs/parent dashboard-path)+ (str/replace (fs/file-name dashboard-path) #"\.ya?ml$" ""))](when (fs/directory? dir)(->> (fs/list-dir dir)(filter yaml-file?)@@ -134,24 +135,21 @@(sort-by :path)(keep (comp :name :doc))))))-(defn- model-columns- "Display names of every column the model exposes."- [doc]- (keep :display_name (:result_metadata doc)))-(defn- dashboard-entries [yamls](for [{:keys [path doc]} yamls:when (dashboard? doc)]{:name (:name doc):description (:description doc)- :cards (dashboard-card-names path)}))+ :label "Cards"+ :items (dashboard-card-names path)}))(defn- model-entries [yamls](for [{:keys [doc]} yamls:when (model? doc)]{:name (:name doc):description (:description doc)- :columns (model-columns doc)}))+ :label "Columns"+ :items (keep :display_name (:result_metadata doc))}));; ---------------------------------------------------------------------------;; Categorical-column extractors@@ -162,34 +160,35 @@keyword names as sorted strings."[src](let [content (slurp src)- m (re-find #"(?s)\(mr/def\s+::context.*?\[:enum(.*?)\]\)" content)]+ ;; Only an optional docstring may sit between `::context` and its `[:enum`, so a+ ;; `::context` that stops being an enum throws below rather than matching some later+ ;; definition's `[:enum` and yielding a plausible-looking wrong list.+ m (re-find #"(?s)\(mr/def\s+::context\s*(?:\"[^\"]*\"\s*)?\[:enum(.*?)\]\)" content)](when-not m(throw (ex-info (str "Could not find (mr/def ::context [:enum ...]) in " src){:source src})))(->> (re-seq #":([a-z][a-z0-9-]*(?:/[a-z][a-z0-9-]*)?)" (second m))(map second)(remove #(str/includes? % "/"))- distinct- sort- vec- (non-empty! "query sources"))))+ (sorted-values "query sources"))))(defn- audit-log-topics- "Grep `(derive :event/<topic> ::<parent>)` from the audit-log event file,- then apply the v_audit_log SQL view's rename and exclusion rules so the- list matches what users see in the analytics model."+ "Grep [[event-derive-re]] over the audit-log event file, then apply the v_audit_log SQL+ view's rename and exclusion rules so the list matches what users see in the analytics+ model."[src]- (let [content (slurp src)- topics (->> (re-seq #"\(derive\s+:event/([a-z0-9-]+)\s+::[a-z0-9-]+\)" content)- (map second)- distinct)]- (->> topics+ (let [forms (re-seq event-derive-re (slurp src))+ ;; Guard against partial drift: `sorted-values` only catches the case where every+ ;; call site changed shape, not where a few did and get silently dropped.+ drifted (keep #(when-not (nth % 2) (second %)) forms)]+ (when (seq drifted)+ (throw (ex-info (str "`derive!` parent is no longer a keyword for: " (str/join ", " drifted))+ {:source src :topics (vec drifted)})))+ (->> forms+ (map second)(remove audit-topic-exclusions)(map #(get audit-topic-renames % %))- distinct- sort- vec- (non-empty! "audit log topics"))))+ (sorted-values "audit log topics"))))(defn- content-entity-types"Pull `'X' AS entity_type` literals from the latest mysql-content.sql view,@@ -203,10 +202,7 @@has-card-arm? (and (re-find #"(?i)type\s+as\s+entity_type" content)(re-find #"(?i)report_card" content))](->> (concat literals (when has-card-arm? content-card-types))- distinct- sort- vec- (non-empty! "content entity types"))))+ (sorted-values "content entity types"))))(defn- assert-view-log-sql-shape!"Throws if the v_view_log SQL view stops using `model AS entity_type` —@@ -217,26 +213,21 @@(throw (ex-info "v_view_log SQL no longer uses `model AS entity_type`. Re-derive view-log-entity-types manually."{:source sql-file})))))-(defn- categorical-sections [{:keys [sources]}]+(defn- categorical-entries+ "The fixed-value column lists, each read from its canonical source."+ [sources][{:name "Activity log topics":description "The Topic column on the [Activity log](#activity-log) model takes one of:"- :values-fn #(audit-log-topics (:audit-log-events sources))}+ :items (audit-log-topics (:audit-log-events sources))}{:name "Query log query sources":description "The Query Source column on the [Query log](#query-log) model takes one of:"- :values-fn #(query-sources (:query-sources sources))}+ :items (query-sources (:query-sources sources))}{:name "Content entity types":description "The Entity Type column on the [Content](#content) model takes one of:"- :values-fn #(content-entity-types (:content-views sources))}+ :items (content-entity-types (:content-views sources))}{:name "View log entity types":description "The Entity Type column on the [View log](#view-log) model takes one of:"- :values-fn (constantly view-log-entity-types)}])--(defn- categorical-entries [sections]- (mapv (fn [{:keys [name description values-fn]}]- {:name name- :description description- :values (values-fn)})- sections))+ :items view-log-entity-types}]);; ---------------------------------------------------------------------------;; Markdown rendering@@ -245,59 +236,19 @@(defn- bullet-list [items](str/join "\n" (map #(str "- " %) items)))-(defn- section-header [level text]- (str (apply str (repeat level "#")) " " text))--(defn- labelled-list-renderer- "Returns a fn that renders items as `<label>:\\n\\n<bullets>`, falling back- to `empty-label` when the seq is empty."- [label empty-label]- (fn [items]- (if (seq items)- (str label ":\n\n" (bullet-list items))- empty-label)))--(defn- bare-list- "Render `items` as a plain bullet list, or `nil` when empty."- [items]- (when (seq items)- (bullet-list items)))--(defn- entity-markdown- "Render one `###` subsection from `{:name :description :items :render-items}`."- [{:keys [name description items render-items]}]- (str/join "\n\n"- (remove str/blank?- [(section-header 3 name)- description- (render-items items)])))--(defn- entities-markdown- [{:keys [items-key render-items]} entities]- (->> entities- (map (fn [e]- (entity-markdown {:name (:name e)- :description (:description e)- :items (items-key e)- :render-items render-items})))+(defn- section-markdown+ "Render one `###` subsection from `{:name :description :label :items}`. The list is+ dropped when `items` is empty, and rendered bare when there is no `label`."+ [{:keys [name description label items]}]+ (->> [(str "### " name)+ description+ (when (seq items)+ (str (when label (str label ":\n\n")) (bullet-list items)))]+ (remove str/blank?)(str/join "\n\n")))-(def ^:private dashboard-section- {:items-key :cards- :render-items (labelled-list-renderer "Cards" "_No cards found._")})--(def ^:private model-section- {:items-key :columns- :render-items (labelled-list-renderer "Columns" "_No columns found._")})--(def ^:private categorical-section- {:items-key :values- :render-items bare-list})--(defn- intro-markdown- "Substitute `{{title}}` into `template-content`."- [template-content title]- (str/replace template-content "{{title}}" title))+(defn- sections-markdown [entries]+ (str/join "\n\n" (map section-markdown entries)))(defn- document-markdown[{:keys [intro dashboards models categorical]}]@@ -305,19 +256,19 @@(remove str/blank?[(str/trimr intro)"## Dashboards"- (entities-markdown dashboard-section dashboards)+ (sections-markdown dashboards)"## Models"- (entities-markdown model-section models)+ (sections-markdown models)"## Categorical column values""Some columns in the models above hold one of a fixed set of values."- (entities-markdown categorical-section categorical)]))+ (sections-markdown categorical)]))"\n"));; ---------------------------------------------------------------------------;; Entry point;; ----------------------------------------------------------------------------(defn- generate! [{:keys [yaml-dir template output title sources] :as cfg}]+(defn- generate! [{:keys [yaml-dir template output sources]}](when-not (fs/directory? yaml-dir)(throw (ex-info (str "YAML directory does not exist: " yaml-dir){:yaml-dir yaml-dir@@ -326,10 +277,9 @@(let [yamls (top-level-yamls yaml-dir)dashboards (dashboard-entries yamls)models (model-entries yamls)- categorical (categorical-entries (categorical-sections cfg))- intro (intro-markdown (slurp template) title)+ categorical (categorical-entries sources)content (document-markdown- {:intro intro+ {:intro (slurp template):dashboards dashboards:models models:categorical categorical})]@@ -340,11 +290,7 @@(count dashboards)(count models)(count categorical)- (str/join ", " (map (comp count :values) categorical))))- {:dashboards (count dashboards)- :models (count models)- :categorical (count categorical)- :output output}))+ (str/join ", " (map (comp count :items) categorical))))))(defn -main [& _args](trydiff --git a/bin/templates/usage-analytics-reference-intro.md b/bin/templates/usage-analytics-reference-intro.mdindex 8c032abf82c..bac0cbad1f8 100644--- a/bin/templates/usage-analytics-reference-intro.md+++ b/bin/templates/usage-analytics-reference-intro.md@@ -1,9 +1,9 @@----title: {{title}}+title: Usage analytics referencesummary: "A list of the dashboards and models included in the Usage Analytics collection."----# {{title}}+# Usage analytics reference_This page is generated by `bin/generate-usage-analytics-docs.bb`._