The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Metaplow Analytics for Internal Metabase Instance (#72380)
Ryan Laurie4c551c71fb06
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex d1a37f28fdc..5fbd9803eab 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -187,9 +187,9 @@{:team "UX West":api #{metabase.analytics.apimetabase.analytics.core+ metabase.analytics.eventmetabase.analytics.init- metabase.analytics.settings- metabase.analytics.snowplow}+ metabase.analytics.settings}:uses #{analytics-interfaceapiapp-dbdiff --git a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj b/.clj-kondo/src/hooks/metabase/settings/models/setting.cljindex d31472af446..d7d5d1e5253 100644--- a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj+++ b/.clj-kondo/src/hooks/metabase/settings/models/setting.clj@@ -95,6 +95,8 @@metabot-feedback-urlmetabot-get-prompt-templates-urlmetabot-prompt-generator-token-limit+ metaplow-tracking-enabled+ metaplow-urlmulti-setting-read-onlynotebook-native-preview-sidebar-widthnotification-link-base-urldiff --git a/enterprise/backend/src/metabase_enterprise/gsheets/api.clj b/enterprise/backend/src/metabase_enterprise/gsheets/api.cljindex 85db9373008..52ae0d206d4 100644--- a/enterprise/backend/src/metabase_enterprise/gsheets/api.clj+++ b/enterprise/backend/src/metabase_enterprise/gsheets/api.clj@@ -8,7 +8,7 @@:refer [gsheets gsheets!]][metabase-enterprise.harbormaster.client :as hm.client][metabase.analytics-interface.core :as analytics]- [metabase.analytics.snowplow :as snowplow]+ [metabase.analytics.event :as analytics.event][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.util :as u]@@ -229,7 +229,7 @@[{} {} {:keys [url]} :- [:map [:url ms/NonBlankString]]](let [attached-dwh (t2/select-one-fn :id :model/Database :is_attached_dwh true)](when-not (some? attached-dwh)- (snowplow/track-event! :snowplow/simple_event {:event "sheets_connected" :event_detail "fail - no dwh"})+ (analytics.event/track-event! :snowplow/simple_event {:event "sheets_connected" :event_detail "fail - no dwh"})(throw-error 400 (tru "No attached dwh found.") nil))(let [[status response] (hm-create-gdrive-conn! url)@@ -359,10 +359,10 @@(let [sheet-config (gsheets-safe)](if (empty? sheet-config)(do- (snowplow/track-event! :snowplow/simple_event {:event "sheets_sync" :event_detail "fail - no config"})+ (analytics.event/track-event! :snowplow/simple_event {:event "sheets_sync" :event_detail "fail - no config"})(throw-error 404 (tru "No attached google sheet(s) found.") nil))(do- (snowplow/track-event! :snowplow/simple_event {:event "sheets_sync"})+ (analytics.event/track-event! :snowplow/simple_event {:event "sheets_sync"})(analytics/inc! :metabase-gsheets/connection-manually-synced)(let [[status response] (hm-sync-conn! (:gdrive/conn-id sheet-config))](if (= status :ok)@@ -378,7 +378,7 @@(api.macros/defendpoint :delete "/connection""Disconnect the google service account. There is only one (or zero) at the time of writing."[]- (snowplow/track-event! :snowplow/simple_event {:event "sheets_disconnected"})+ (analytics.event/track-event! :snowplow/simple_event {:event "sheets_disconnected"})(analytics/inc! :metabase-gsheets/connection-deleted)(reset-gsheets-status!){:status "not-connected"})diff --git a/enterprise/backend/src/metabase_enterprise/security_center/models/security_advisory.clj b/enterprise/backend/src/metabase_enterprise/security_center/models/security_advisory.cljindex a7ec4348d58..3a998abd2b5 100644--- a/enterprise/backend/src/metabase_enterprise/security_center/models/security_advisory.clj+++ b/enterprise/backend/src/metabase_enterprise/security_center/models/security_advisory.clj@@ -1,6 +1,6 @@(ns metabase-enterprise.security-center.models.security-advisory(:require- [metabase.analytics.snowplow :as snowplow]+ [metabase.analytics.core :as analytics][metabase.events.core :as events][metabase.models.interface :as mi][methodical.core :as methodical]@@ -49,9 +49,9 @@(events/publish-event! :event/security-advisory-acknowledge{:object advisory:user-id user-id})- (snowplow/track-event! :snowplow/simple_event- {:event "security_advisory_acknowledged"- :event_detail (name (:severity advisory))})+ (analytics/track-event! :snowplow/simple_event+ {:event "security_advisory_acknowledged"+ :event_detail (name (:severity advisory))})(-> (t2/select-one :model/SecurityAdvisory :id (:id advisory))(t2/hydrate :acknowledged_by_user))))diff --git a/enterprise/backend/src/metabase_enterprise/security_center/notification.clj b/enterprise/backend/src/metabase_enterprise/security_center/notification.cljindex d5b68ff159c..088c8944f64 100644--- a/enterprise/backend/src/metabase_enterprise/security_center/notification.clj+++ b/enterprise/backend/src/metabase_enterprise/security_center/notification.clj@@ -10,7 +10,7 @@included as a recipient when set."(:require[metabase-enterprise.security-center.settings :as settings]- [metabase.analytics.snowplow :as snowplow]+ [metabase.analytics.core :as analytics][metabase.channel.settings :as channel.settings][metabase.events.core :as events][metabase.models.interface :as mi]@@ -88,11 +88,11 @@"Track a Snowplow event for each channel in the notification's handlers."[notification triggered-from result](doseq [{:keys [channel_type]} (:handlers notification)]- (snowplow/track-event! :snowplow/simple_event- {:event "security_advisory_notification_sent"- :event_detail (channel-type->name channel_type)- :triggered_from triggered-from- :result result})))+ (analytics/track-event! :snowplow/simple_event+ {:event "security_advisory_notification_sent"+ :event_detail (channel-type->name channel_type)+ :triggered_from triggered-from+ :result result})))(def ^:private test-advisory"A synthetic advisory used for test notifications so admins can verify delivery."diff --git a/frontend/src/metabase-types/api/mocks/settings.ts b/frontend/src/metabase-types/api/mocks/settings.tsindex 00d721a0be6..88a8e7332ad 100644--- a/frontend/src/metabase-types/api/mocks/settings.ts+++ b/frontend/src/metabase-types/api/mocks/settings.ts@@ -175,6 +175,8 @@ export const createMockSettings = ("custom-viz-plugin-dev-mode-enabled": false,"allowed-iframe-hosts": "*","anon-tracking-enabled": false,+ "metaplow-tracking-enabled": false,+ "metaplow-url": null,"application-colors": {},"application-font": "Lato","application-font-files": [],diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.tsindex 74afea6c26c..2d2a118ed32 100644--- a/frontend/src/metabase-types/api/settings.ts+++ b/frontend/src/metabase-types/api/settings.ts@@ -551,6 +551,8 @@ interface PublicSettings {"agent-api-enabled?": boolean;"analytics-uuid": string;"anon-tracking-enabled": boolean;+ "metaplow-tracking-enabled": boolean;+ "metaplow-url": string | null;"application-font": string;"application-font-files": FontFile[] | null;"application-name": string;diff --git a/frontend/src/metabase/analytics/event.ts b/frontend/src/metabase/analytics/event.tsindex 65da66f6915..98aeb23756d 100644--- a/frontend/src/metabase/analytics/event.ts+++ b/frontend/src/metabase/analytics/event.ts@@ -1,6 +1,7 @@import * as Snowplow from "@snowplow/browser-tracker";import { shouldLogAnalytics } from "metabase/env";+import { trackMetaplowEvent } from "metabase/utils/metaplow";import Settings from "metabase/utils/settings";import type { SchemaEventMap, SchemaType } from "metabase-types/analytics";import type { SimpleEventSchema } from "metabase-types/analytics/event";@@ -43,14 +44,14 @@ export function trackSchemaEvent<S extends SchemaType>(schema: S,event: SchemaEventMap[S],): void {- const shouldSendEvent =- Settings.trackingEnabled() && Settings.snowplowEnabled();+ const trackingEnabled = Settings.trackingEnabled();+ const shouldSendSnowplow = Settings.snowplowEnabled();if (shouldLogAnalytics) {const { event: type, ...other } = event;// eslint-disable-next-line no-consoleconsole.log(- `%c[SNOWPLOW EVENT | event sent:${shouldSendEvent}]%c, ${type}`,+ `%c[SNOWPLOW EVENT | event sent:${shouldSendSnowplow}]%c, ${type}`,// eslint-disable-next-line metabase/no-color-literals"background: #222; color: #bada55","color: ",@@ -58,7 +59,11 @@ export function trackSchemaEvent<S extends SchemaType>();}- if (shouldSendEvent) {+ if (!trackingEnabled) {+ return;+ }++ if (shouldSendSnowplow) {Snowplow.trackSelfDescribingEvent({event: {schema: `iglu:com.metabase/${schema}/jsonschema/${VERSIONS[schema]}`,@@ -66,4 +71,9 @@ export function trackSchemaEvent<S extends SchemaType>(},});}++ if (Settings.get("metaplow-tracking-enabled")) {+ const { event: name, ...data } = event;+ trackMetaplowEvent(name, data);+ }}diff --git a/frontend/src/metabase/analytics/page-view.ts b/frontend/src/metabase/analytics/page-view.tsindex bc0a50d8ef2..ee856673bc6 100644--- a/frontend/src/metabase/analytics/page-view.ts+++ b/frontend/src/metabase/analytics/page-view.ts@@ -1,15 +1,22 @@import * as Snowplow from "@snowplow/browser-tracker";+import { trackMetaplowPageView } from "metabase/utils/metaplow";import Settings from "metabase/utils/settings";export const trackPageView = (url: string): void => {- if (!url || !Settings.trackingEnabled() || !Settings.snowplowEnabled()) {+ if (!url || !Settings.trackingEnabled()) {return;}- Snowplow.setReferrerUrl("#");- Snowplow.setCustomUrl(getSanitizedUrl(url));- Snowplow.trackPageView();+ if (Settings.snowplowEnabled()) {+ Snowplow.setReferrerUrl("#");+ Snowplow.setCustomUrl(getSanitizedUrl(url));+ Snowplow.trackPageView();+ }++ if (Settings.get("metaplow-tracking-enabled")) {+ trackMetaplowPageView(url);+ }};const getSanitizedUrl = (url: string): string => {diff --git a/frontend/src/metabase/utils/metaplow.ts b/frontend/src/metabase/utils/metaplow.tsnew file mode 100644index 00000000000..eac8cec26d2--- /dev/null+++ b/frontend/src/metabase/utils/metaplow.ts@@ -0,0 +1,77 @@+import Settings from "metabase/utils/settings";++const METAPLOW_WEBSITE_ID = "23eefa30-4c4f-490e-aa4f-084cd23b1561";+const anonymizedHostname = "anonymous.metabase.com";+const anonymizedOrigin = `http://${anonymizedHostname}`;++type MetaplowPayload = {+ website: string;+ url: string;+ id: string;+ referrer: string;+ title: string;+ hostname: string;+ screen: string;+ language: string;+ tag: string;+ name: string;+ data?: Record<string, unknown>;+};++const getSanitizedUrl = (url: string) => {+ const parsed = new URL(url, window.location.origin);+ const pathWithoutSlug = parsed.pathname.replace(+ /(\/\d+)-[^\/]+$/,+ (_, path) => path,+ );+ return anonymizedOrigin + pathWithoutSlug;+};++function getBasePayload(url: string): Omit<MetaplowPayload, "name" | "data"> {+ return {+ website: METAPLOW_WEBSITE_ID,+ url: getSanitizedUrl(url),+ id: Settings.get("analytics-uuid") ?? "",+ referrer: "",+ title: "",+ hostname: anonymizedHostname,+ screen: `${window.screen.width}x${window.screen.height}`,+ language: navigator.language,+ tag: "metabase-instance",+ };+}++async function send(payload: MetaplowPayload): Promise<unknown> {+ const metaplowUrl = Settings.get("metaplow-url");+ if (!metaplowUrl) {+ return;+ }++ const headers: Record<string, string> = {+ "Content-Type": "application/json",+ };++ return fetch(metaplowUrl, {+ method: "POST",+ headers,+ body: JSON.stringify({ type: "event", payload }),+ });+}++export function trackMetaplowEvent(+ name: MetaplowPayload["name"],+ data: MetaplowPayload["data"] = {},+): void {+ send({+ ...getBasePayload(window.location.href),+ name,+ data,+ }).catch(() => undefined);+}++export function trackMetaplowPageView(url: string): Promise<unknown> {+ return send({+ ...getBasePayload(url),+ name: "pageview",+ }).catch(() => undefined);+}diff --git a/frontend/src/metabase/utils/metaplow.unit.spec.ts b/frontend/src/metabase/utils/metaplow.unit.spec.tsnew file mode 100644index 00000000000..4ec2b562745--- /dev/null+++ b/frontend/src/metabase/utils/metaplow.unit.spec.ts@@ -0,0 +1,123 @@+import Settings from "metabase/utils/settings";++import { trackMetaplowEvent, trackMetaplowPageView } from "./metaplow";++const METAPLOW_URL = "https://metaplow.example.com/api/send";+const METAPLOW_WEBSITE_ID = "23eefa30-4c4f-490e-aa4f-084cd23b1561";+const ANON_ORIGIN = "http://anonymous.metabase.com";++describe("metaplow", () => {+ let fetchSpy: jest.SpyInstance;++ beforeEach(() => {+ Settings.set("metaplow-url", METAPLOW_URL);+ fetchSpy = jest+ .spyOn(global, "fetch")+ .mockResolvedValue(new Response(null, { status: 200 }));+ });++ afterEach(() => {+ fetchSpy.mockRestore();+ Settings.set("metaplow-url", null);+ });++ const getSentPayload = () => {+ expect(fetchSpy).toHaveBeenCalledTimes(1);+ const [url, init] = fetchSpy.mock.calls[0];+ expect(url).toBe(METAPLOW_URL);+ expect(init?.method).toBe("POST");+ expect((init?.headers as Record<string, string>)["Content-Type"]).toBe(+ "application/json",+ );+ return JSON.parse(init?.body as string);+ };++ describe("trackMetaplowEvent", () => {+ it("does not call fetch when metaplow-url is not set", () => {+ Settings.set("metaplow-url", null);+ trackMetaplowEvent("button_clicked");+ expect(fetchSpy).not.toHaveBeenCalled();+ });++ it("sends an event with the given name and default empty data", () => {+ trackMetaplowEvent("button_clicked");++ const body = getSentPayload();+ expect(body.type).toBe("event");+ expect(body.payload).toEqual(+ expect.objectContaining({+ website: METAPLOW_WEBSITE_ID,+ name: "button_clicked",+ data: {},+ referrer: "",+ title: "",+ hostname: "anonymous.metabase.com",+ tag: "metabase-instance",+ }),+ );+ });++ it("forwards the data payload", () => {+ trackMetaplowEvent("button_clicked", { foo: "bar", count: 3 });++ const { payload } = getSentPayload();+ expect(payload.data).toEqual({ foo: "bar", count: 3 });+ });++ it("includes screen and language from the browser", () => {+ trackMetaplowEvent("button_clicked");++ const { payload } = getSentPayload();+ expect(payload.screen).toBe(+ `${window.screen.width}x${window.screen.height}`,+ );+ expect(payload.language).toBe(navigator.language);+ });++ it("uses an anonymized hostname in the url", () => {+ trackMetaplowEvent("button_clicked");++ const { payload } = getSentPayload();+ expect(payload.url.startsWith(ANON_ORIGIN)).toBe(true);+ });+ });++ describe("trackMetaplowPageView", () => {+ it("does not call fetch when metaplow-url is not set", async () => {+ Settings.set("metaplow-url", null);+ await trackMetaplowPageView("/question/42-my-question");+ expect(fetchSpy).not.toHaveBeenCalled();+ });++ it('sends an event with name "pageview"', async () => {+ await trackMetaplowPageView("/dashboard/1");++ const { payload } = getSentPayload();+ expect(payload.name).toBe("pageview");+ expect(payload.url).toBe(`${ANON_ORIGIN}/dashboard/1`);+ });++ it("strips slugs from /:id-slug paths", async () => {+ await trackMetaplowPageView("/question/42-my-favorite-question");++ const { payload } = getSentPayload();+ expect(payload.url).toBe(`${ANON_ORIGIN}/question/42`);+ });++ it("anonymizes absolute URLs by replacing the origin", async () => {+ await trackMetaplowPageView(+ "https://my-company.metabaseapp.com/collection/5-secrets",+ );++ const { payload } = getSentPayload();+ expect(payload.url).toBe(`${ANON_ORIGIN}/collection/5`);+ });++ it("preserves paths without a numeric slug prefix", async () => {+ await trackMetaplowPageView("/admin/settings/general");++ const { payload } = getSentPayload();+ expect(payload.url).toBe(`${ANON_ORIGIN}/admin/settings/general`);+ });+ });+});diff --git a/src/metabase/analytics/core.clj b/src/metabase/analytics/core.cljindex 05f118644eb..56dce821119 100644--- a/src/metabase/analytics/core.clj+++ b/src/metabase/analytics/core.clj@@ -1,22 +1,22 @@(ns metabase.analytics.core(:require+ [metabase.analytics.event][metabase.analytics.llm-token-usage][metabase.analytics.prometheus][metabase.analytics.quartz][metabase.analytics.sdk][metabase.analytics.settings]- [metabase.analytics.snowplow][metabase.analytics.stats][metabase.analytics.util][potemkin :as p]))(comment+ metabase.analytics.event/keep-memetabase.analytics.llm-token-usage/keep-memetabase.analytics.prometheus/keep-memetabase.analytics.quartz/keep-memetabase.analytics.sdk/keep-memetabase.analytics.settings/keep-me- metabase.analytics.snowplow/keep-memetabase.analytics.stats/keep-memetabase.analytics.util/keep-me)@@ -65,7 +65,7 @@anon-tracking-enabled!instance-creation]- [metabase.analytics.snowplow+ [metabase.analytics.eventtrack-event!]diff --git a/src/metabase/analytics/event.clj b/src/metabase/analytics/event.cljnew file mode 100644index 00000000000..e5af10a60e2--- /dev/null+++ b/src/metabase/analytics/event.clj@@ -0,0 +1,17 @@+(ns metabase.analytics.event+ "Dispatches analytics events to Snowplow and Metaplow. Each tracker gates itself on its own setting; this namespace+ just fans the call out."+ (:require+ [metabase.analytics.metaplow :as metaplow]+ [metabase.analytics.snowplow :as snowplow]+ [metabase.api.common :as api]))++(defn track-event!+ "Send a single analytics event to Snowplow and Metaplow. Each tracker decides whether to emit based on its own+ setting."+ ([schema data]+ (track-event! schema data api/*current-user-id*))+ ([schema data user-id]+ (let [snowplow? (snowplow/track-event! schema data user-id)+ metaplow? (metaplow/track-event! schema data user-id)]+ (or snowplow? metaplow?))))diff --git a/src/metabase/analytics/llm_token_usage.clj b/src/metabase/analytics/llm_token_usage.cljindex d062fb169ac..74eb619faf0 100644--- a/src/metabase/analytics/llm_token_usage.clj+++ b/src/metabase/analytics/llm_token_usage.clj@@ -2,7 +2,7 @@"LLM token usage tracking for Snowplow and Prometheus."(:require[metabase.analytics-interface.core :as analytics]- [metabase.analytics.snowplow :as snowplow]+ [metabase.analytics.event :as analytics.event][metabase.analytics.util :as analytics.util][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]))@@ -34,24 +34,24 @@estimated-costs-usd user-id duration-ms source tag session-id profilehashed-metabase-license-token]}:- SnowplowArgs]- (snowplow/track-event! :snowplow/token_usage- {:hashed-metabase-license-token (or hashed-metabase-license-token- (analytics.util/hashed-metabase-token-or-uuid))- :request-id request-id- :model-id model-id- :total-tokens total-tokens- :prompt-tokens prompt-tokens- :completion-tokens completion-tokens- :cache-creation-tokens cache-creation-tokens- :cache-read-tokens cache-read-tokens- :estimated-costs-usd estimated-costs-usd- :user-id user-id- :duration-ms (some-> duration-ms long)- :source source- :tag tag- :session-id session-id- :profile profile}- user-id))+ (analytics.event/track-event! :snowplow/token_usage+ {:hashed-metabase-license-token (or hashed-metabase-license-token+ (analytics.util/hashed-metabase-token-or-uuid))+ :request-id request-id+ :model-id model-id+ :total-tokens total-tokens+ :prompt-tokens prompt-tokens+ :completion-tokens completion-tokens+ :cache-creation-tokens cache-creation-tokens+ :cache-read-tokens cache-read-tokens+ :estimated-costs-usd estimated-costs-usd+ :user-id user-id+ :duration-ms (some-> duration-ms long)+ :source source+ :tag tag+ :session-id session-id+ :profile profile}+ user-id))(def ^:private PrometheusArgs[:mapdiff --git a/src/metabase/analytics/metaplow.clj b/src/metabase/analytics/metaplow.cljnew file mode 100644index 00000000000..8816b0710da--- /dev/null+++ b/src/metabase/analytics/metaplow.clj@@ -0,0 +1,201 @@+(ns metabase.analytics.metaplow+ "Sends analytics events to a Metaplow (Umami-compatible) collector.++ `track-event!` builds a payload, offers it onto an async send queue, and returns. A `pipeline-blocking` worker pool+ drains the queue and POSTs each event to the collector's `/api/send` endpoint, retrying transient failures via+ [[metabase.util.retry/with-retry]]. The HTTP connection pool is sized to match the worker count so workers never+ contend for slots.++ Emission is gated by [[metabase.analytics.settings/metaplow-tracking-enabled]]. The public entry point used by the+ rest of the codebase is [[metabase.analytics.event/track-event!]], which fans out to both Snowplow and Metaplow."+ (:require+ [clj-http.client :as http]+ [clj-http.conn-mgr :as conn-mgr]+ [clojure.core.async :as a]+ [clojure.walk :as walk]+ [metabase.analytics-interface.core :as analytics]+ [metabase.analytics.settings :as analytics.settings]+ [metabase.premium-features.core :as premium-features]+ [metabase.util :as u]+ [metabase.util.json :as json]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.retry :as retry]+ [metabase.version.core :as version]))++(set! *warn-on-reflection* true)++(def ^:private website-id+ "Same constant the frontend tracker uses; see frontend/src/metabase/utils/metaplow.ts."+ "23eefa30-4c4f-490e-aa4f-084cd23b1561")++(def ^:private anonymized-hostname+ "Hostname sent on every event. Matches the FE constant in frontend/src/metabase/utils/metaplow.ts."+ "anonymous.metabase.com")++(def ^:private payload-tag+ "Tag identifying the event source. Matches the FE constant in frontend/src/metabase/utils/metaplow.ts."+ "metabase-instance")++(def ^:private queue-size 10000)+(def ^:private no-retry-status-codes #{400 401 403 410 422})++;; `worker-count` drives both the number of worker threads draining the queue AND the size of the HTTP connection+;; pool. Keep them coupled: if the pool is smaller than the worker count, workers contend for connection slots and+;; the parallelism is silently capped at the pool size.+(def ^:private worker-count 50)++(defn- normalize-kw+ [kw]+ (name (u/->snake_case_en kw)))++(defn- snake-case-payload+ "Convert any keyword in a payload (keys or values, at any depth) to a snake_case string."+ [m]+ (walk/postwalk #(if (keyword? %) (normalize-kw %) %) m))++(defn- event-name+data+ "Returns `[event-name event-data]` for a given Snowplow-style schema kw and payload. Schemas with an `:event`+ discriminator become `\"<schema>.<event>\"`; schemas without one become `\"<schema>\"`."+ [schema data]+ (let [schema-name (normalize-kw schema)+ data* (snake-case-payload data)]+ (if-let [event (get data* "event")]+ [(str schema-name "." event) (dissoc data* "event")]+ [schema-name data*])))++(defn- build-payload+ [schema data]+ (let [[event-name event-data] (event-name+data schema data)+ enriched-data (assoc event-data+ "version_tag" (:tag (version/version))+ "plan" (or (premium-features/plan-alias) "oss"))]+ {:type "event"+ :payload {:website website-id+ :id (analytics.settings/analytics-uuid)+ :hostname anonymized-hostname+ :tag payload-tag+ :name event-name+ :data enriched-data}}))++(defonce ^:private+ ^{:doc "Reuses TCP/TLS connections across requests to the metaplow collector. Same idea as Snowplow's+ `PoolingHttpClientConnectionManager` setup in `metabase.analytics.snowplow`, just via clj-http's wrapper.+ Sized to `worker-count`: every request goes to one host, so `:default-per-route` is the actual parallelism+ cap. Wrapped in `delay` so the pool isn't created until the first event is sent."}+ connection-manager+ (delay (conn-mgr/make-reusable-conn-manager {:threads worker-count+ :default-per-route worker-count})))++(defn- send-event!+ "POST a single payload to the Metaplow `/api/send` endpoint. Returns a map with `:status` (HTTP status code, or -1+ on connection failure)."+ [payload]+ (try+ (http/post (analytics.settings/metaplow-url)+ {:body (json/encode payload)+ :content-type :json+ :socket-timeout 5000+ :connection-timeout 5000+ :throw-exceptions false+ :connection-manager @connection-manager})+ (catch Throwable e+ (analytics/inc! :metabase-metaplow/errors {:stage :send-event!})+ (log/warn e "Connection failure sending Metaplow event")+ {:status -1})))++(defn- retryable-response?+ "Truthy when a `send-event!` response should trigger another attempt: non-2xx and not one of the no-retry+ status codes that Snowplow also gives up on (400/401/403/410/422)."+ [{:keys [status]}]+ (and (some? status)+ (not (<= 200 status 299))+ (not (contains? no-retry-status-codes status))))++(defn- send-event-with-retries!+ "Send an event, retrying with exponential backoff on connection failures or retryable HTTP statuses (via+ [[metabase.util.retry/with-retry]]). Returns `:sent` on success and `:error` on failure (never nil), so callers can+ reliably distinguish outcomes."+ [payload]+ (try+ (retry/with-retry {:retry-if (fn [response _ex] (retryable-response? response))}+ (send-event! payload))+ :sent+ (catch Throwable e+ (analytics/inc! :metabase-metaplow/errors {:stage :send-event-with-retries!})+ (log/warn e "Error sending Metaplow event")+ :error)))++(defonce ^:private+ ^{:doc "Individual payloads queued by `track-event!` land here. Bounded chan of size `queue-size`; new events are+ rejected when full."}+ source-chan+ (a/chan queue-size))++(defonce ^:private+ ^{:doc "Black-hole sink for `pipeline-blocking` worker output. Sliding-buffer with no consumer, so workers' puts+ never block."}+ sink-chan+ (a/chan (a/sliding-buffer 1)))++(defonce ^:private+ ^{:doc "Spawns `worker-count` worker threads that drain `source-chan` and apply `send-event-with-retries!` to each+ event. Wrapped in `delay` so workers aren't spawned until the first event is enqueued — instances that don't+ use Metaplow pay nothing at boot."}+ pipeline+ (delay+ (a/pipeline-blocking+ worker-count+ sink-chan+ ;; The var (not the fn) is passed so the indirection survives `with-redefs` in tests.+ (map #'send-event-with-retries!)+ source-chan)))++(defn- enqueue!+ "Push a payload onto the async send queue. Returns true on success, false when the queue is full."+ [payload]+ @pipeline ;; start the pipeline on first call+ (or (a/offer! source-chan payload)+ (do (analytics/inc! :metabase-metaplow/errors {:stage :enqueue!})+ false)))++(mu/defn track-event! :- :boolean+ "Send a single analytics event to the Metaplow collector. Returns true when the event was enqueued, false when+ Metaplow tracking is disabled or the queue is full."+ ([schema data]+ (track-event! schema data nil))+ ([schema :- :keyword data _user-id]+ (boolean+ (when (analytics.settings/metaplow-tracking-enabled)+ (try+ (enqueue! (build-payload schema data))+ (catch Throwable e+ (analytics/inc! :metabase-metaplow/errors {:stage :track-event!})+ (log/warn e "Error queueing Metaplow event")+ false))))))++(comment+ ;; Manual testing against a real Umami/Metaplow collector. Eval forms in the REPL one at a time.++ ;; 1. Point at the collector.+ (analytics.settings/metaplow-url! "https://product-analytics-ingestion.staging.metabase.com/api/send")+ (analytics.settings/anon-tracking-enabled! true)++ ;; 2. Sanity check.+ (analytics.settings/metaplow-tracking-enabled) ; => true++ ;; 3. Fire one event: queued, drained by a worker, POSTed to `metaplow-url`.+ (track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id 1})++ ;; 4. Fire a burst: workers POST in parallel up to `worker-count`.+ (dotimes [i 75]+ (track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id i}))++ ;; 5. Inspect what gets built without sending anything.+ (build-payload :snowplow/dashboard {:event :dashboard-created :dashboard-id 42})++ ;; 6. POST a hand-crafted event directly. Bypasses the queue; useful for testing URL, auth, and payload shape.+ (send-event! (build-payload :snowplow/dashboard {:event :test-event :dashboard-id 1}))++ ;; 7. Turn it off.+ (analytics.settings/metaplow-url! nil))diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex bec9452d237..6bfa7255f13 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -690,7 +690,12 @@{:description "Unix timestamp (seconds since epoch) of the last successful Security Center."})(prometheus/gauge :metabase-security-center/vulnerable-advisories{:description "Number of advisories where this Metabase instance is potentially affected."- :labels [:severity :acknowledged]})])+ :labels [:severity :acknowledged]})++ ;; metaplow analytics metrics+ (prometheus/counter :metabase-metaplow/errors+ {:description "Metaplow event pipeline errors by stage."+ :labels [:stage]})])(defn- quartz-collectors[]diff --git a/src/metabase/analytics/settings.clj b/src/metabase/analytics/settings.cljindex 1b7713f8732..d14be069769 100644--- a/src/metabase/analytics/settings.clj+++ b/src/metabase/analytics/settings.clj@@ -44,6 +44,23 @@:visibility :public:doc false)+(defsetting metaplow-tracking-enabled+ (deferred-tru+ (str "Boolean indicating whether analytics events are being sent to Metaplow. "+ "True if anonymous tracking is enabled for this instance, and a Metaplow collector URL is set."))+ :type :boolean+ :getter (fn [] (boolean (and (anon-tracking-enabled)+ (setting/get-value-of-type :string :metaplow-url))))+ :visibility :public+ :doc false)++(defsetting metaplow-url+ (deferred-tru "The URL of the Metaplow collector to send analytics events to.")+ :encryption :no+ :visibility :public+ :audit :never+ :doc false)+(defsetting snowplow-url(deferred-tru "The URL of the Snowplow collector to send analytics events to."):encryption :no@@ -67,7 +84,7 @@;; is first read.(let [value (or (first-user-creation) (t/offset-date-time))](setting/set-value-of-type! :timestamp :instance-creation value)- ((requiring-resolve 'metabase.analytics.snowplow/track-event!) :snowplow/account {:event :new_instance_created} nil)))+ ((requiring-resolve 'metabase.analytics.event/track-event!) :snowplow/account {:event :new_instance_created} nil)))(u.date/format-rfc3339 (setting/get-value-of-type :timestamp :instance-creation)))(defsetting instance-creationdiff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.cljindex 8ed4caed580..da701d63329 100644--- a/src/metabase/analytics/stats.clj+++ b/src/metabase/analytics/stats.clj@@ -9,8 +9,8 @@[environ.core :as env][java-time.api :as t][medley.core :as m]+ [metabase.analytics.event :as analytics.event][metabase.analytics.settings :as analytics.settings]- [metabase.analytics.snowplow :as snowplow][metabase.app-db.core :as app-db][metabase.appearance.core :as appearance][metabase.config.core :as config]@@ -1048,5 +1048,5 @@(str "Missing required keys in snowplow-data. got:" (sort (keys snowplow-data))))#_{:clj-kondo/ignore [:deprecated-var]}(send-stats-deprecated! stats)- (snowplow/track-event! :snowplow/instance_stats snowplow-data)+ (analytics.event/track-event! :snowplow/instance_stats snowplow-data)(stats-post-cleanup))))diff --git a/src/metabase/llm/api.clj b/src/metabase/llm/api.cljindex 645b04a53fd..2364a5dc341 100644--- a/src/metabase/llm/api.clj+++ b/src/metabase/llm/api.clj@@ -4,7 +4,6 @@[clojure.java.io :as io][clojure.set :as set][metabase.analytics.core :as analytics]- [metabase.analytics.snowplow :as snowplow][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]]@@ -80,12 +79,12 @@(defn- track-sqlgen-event!"Track SQL generation usage via Snowplow simple_event."[{:keys [duration-ms result engine]}]- (snowplow/track-event! :snowplow/simple_event- {:event "metabot_oss_sqlgen_used"- :event_detail (some-> engine name)- :duration_ms (some-> duration-ms long)- :result result}- api/*current-user-id*))+ (analytics/track-event! :snowplow/simple_event+ {:event "metabot_oss_sqlgen_used"+ :event_detail (some-> engine name)+ :duration_ms (some-> duration-ms long)+ :result result}+ api/*current-user-id*))(api.macros/defendpoint :get "/list-models":- [:map [:models [:sequential [:mapdiff --git a/src/metabase/logger/api.clj b/src/metabase/logger/api.cljindex eebb13f053e..cb556f7ecb7 100644--- a/src/metabase/logger/api.clj+++ b/src/metabase/logger/api.clj@@ -3,7 +3,7 @@(:require[clojure.string :as str][flatland.ordered.map :as ordered-map]- [metabase.analytics.snowplow :as snowplow]+ [metabase.analytics.core :as analytics][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.logger.core :as logger]@@ -224,10 +224,10 @@(when-let [task @log-adjustment](cancel-undo-task! task))(let [plan (do (if (empty? log-levels)- (snowplow/track-event! :snowplow/simple_event {:event "log_adjustments_reset"})- (snowplow/track-event! :snowplow/simple_event {:event "log_adjustments_set"- :event_detail (->seconds-str duration_unit- duration)}))+ (analytics/track-event! :snowplow/simple_event {:event "log_adjustments_reset"})+ (analytics/track-event! :snowplow/simple_event {:event "log_adjustments_set"+ :event_detail (->seconds-str duration_unit+ duration)}))(set-log-levels! (update-vals log-levels keyword)))](reset! log-adjustment {:plan plan, :undo-task (undo-task plan duration duration_unit)})))nil)@@ -241,7 +241,7 @@[](api/check-superuser)(when-let [task @log-adjustment]- (snowplow/track-event! :snowplow/simple_event {:event "log_adjustments_reset"})+ (analytics/track-event! :snowplow/simple_event {:event "log_adjustments_reset"})(cancel-undo-task! task)(reset! log-adjustment nil))nil)diff --git a/src/metabase/login_history/record.clj b/src/metabase/login_history/record.cljindex b646beed511..0cfabd40859 100644--- a/src/metabase/login_history/record.clj+++ b/src/metabase/login_history/record.clj@@ -1,6 +1,6 @@(ns metabase.login-history.record(:require- [metabase.analytics.snowplow :as snowplow]+ [metabase.analytics.core :as analytics][metabase.channel.email.messages :as messages][metabase.login-history.models.login-history :as login-history][metabase.login-history.settings :as login-history.settings]@@ -41,4 +41,4 @@(when-not (or (:embedded device-info) (:token_exchange device-info))(maybe-send-login-from-new-device-email history-entry))(when-not (:last_login user)- (snowplow/track-event! :snowplow/account {:event :new-user-created} (u/the-id user)))))+ (analytics/track-event! :snowplow/account {:event :new-user-created} (u/the-id user)))))diff --git a/src/metabase/server/middleware/security.clj b/src/metabase/server/middleware/security.cljindex c89fb369207..236fe3b8932 100644--- a/src/metabase/server/middleware/security.clj+++ b/src/metabase/server/middleware/security.clj@@ -187,6 +187,8 @@;; Snowplow analytics(when (analytics/anon-tracking-enabled)(setting/get-value-of-type :string :snowplow-url))+ (when (analytics/anon-tracking-enabled)+ (setting/get-value-of-type :string :metaplow-url));; Webpack dev server(when config/is-dev?(str "*:" frontend-dev-port " ws://*:" frontend-dev-port))diff --git a/test/metabase/analytics/metaplow_test.clj b/test/metabase/analytics/metaplow_test.cljnew file mode 100644index 00000000000..ac392ca1d5a--- /dev/null+++ b/test/metabase/analytics/metaplow_test.clj@@ -0,0 +1,104 @@+(ns metabase.analytics.metaplow-test+ (:require+ [clojure.test :refer :all]+ [metabase.analytics.metaplow :as metaplow]+ [metabase.analytics.settings :as analytics.settings]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.version.core :as version])+ (:import+ (java.util.concurrent CountDownLatch TimeUnit)))++(set! *warn-on-reflection* true)++(use-fixtures :once (fixtures/initialize :db))++(deftest build-payload-test+ (testing "Schemas with an `:event` key produce `<schema>.<event>` and the event key is removed from data"+ (let [payload (#'metaplow/build-payload :snowplow/dashboard+ {:event :dashboard-created :dashboard-id 42 :num-tabs 1})]+ (is (=? {:type "event"+ :payload {:name "dashboard.dashboard_created"+ :data {"dashboard_id" 42+ "num_tabs" 1}}}+ payload))+ (is (not (contains? (get-in payload [:payload :data]) "event")))))++ (testing "Schemas without an `:event` key produce `<schema>` and keep the full payload"+ (is (=? {:payload {:name "instance_stats"+ :data {"metric_one" 1+ "metric_two" 2}}}+ (#'metaplow/build-payload :snowplow/instance_stats {:metric_one 1 :metric_two 2}))))++ (testing "Top-level payload has the expected keys: website / id / hostname / tag / name / data"+ (let [{:keys [payload]} (#'metaplow/build-payload :snowplow/dashboard {:event :dashboard-created})]+ (is (= #{:website :id :hostname :tag :name :data} (set (keys payload))))+ (is (=? {:website string?+ :id (analytics.settings/analytics-uuid)+ :hostname "anonymous.metabase.com"+ :tag "metabase-instance"}+ payload))))++ (testing "`data` is enriched with version_tag and plan on every event"+ (is (=? {:payload {:data {"version_tag" (:tag (version/version))+ "plan" string?}}}+ (#'metaplow/build-payload :snowplow/dashboard {:event :dashboard-created}))))++ (testing "Keyword keys and values become snake_case strings"+ (is (=? {:payload {:name "database.database_connection_successful"+ :data {"database" "postgres"+ "database_id" 1+ "source" "admin"}}}+ (#'metaplow/build-payload :snowplow/database+ {:event :database-connection-successful+ :database :postgres+ :database-id 1+ :source :admin}))))++ (testing "hostname matches the FE's anonymized constant regardless of site-url"+ (mt/with-temporary-setting-values [site-url "https://stats.metabase.com/"]+ (is (=? {:payload {:hostname "anonymous.metabase.com"}}+ (#'metaplow/build-payload :snowplow/dashboard {:event :dashboard-created}))))))++(deftest tracking-disabled-test+ (testing "When metaplow-tracking-enabled is false the event is not enqueued and the call returns false"+ (mt/with-temporary-setting-values [metaplow-url nil]+ (let [collector (atom [])]+ (with-redefs [metaplow/enqueue! (fn [payload]+ (swap! collector conj payload)+ true)]+ (is (false? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created})))+ (is (empty? @collector)))))))++(deftest pipeline-integration-test+ (mt/with-temporary-setting-values [metaplow-url "http://fake-metaplow/api/send"+ anon-tracking-enabled true]+ (testing "track-event! enqueues onto the real channel and the pipeline worker invokes send-event-with-retries!"+ (let [received (promise)]+ (with-redefs [metaplow/send-event-with-retries! (fn [payload]+ (deliver received payload)+ :sent)]+ (is (true? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id 7})))+ (let [payload (deref received 1000 ::timeout)]+ (is (not= ::timeout payload)+ "Worker did not consume the event within 1s")+ (when (not= ::timeout payload)+ (is (=? {:type "event"+ :payload {:name "dashboard.dashboard_created"+ :data {"dashboard_id" 7}}}+ payload)))))))++ (testing "Sending 200 events: all of them traverse the pipeline"+ (let [received (atom [])+ latch (CountDownLatch. 200)]+ (with-redefs [metaplow/send-event-with-retries! (fn [payload]+ (swap! received conj payload)+ (.countDown latch)+ :sent)]+ (doseq [i (range 200)]+ (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id i}))+ (is (true? (.await latch 1 TimeUnit/SECONDS))+ "Pipeline did not process all 200 events within 1s")+ (is (= 200 (count @received)))+ (is (= (set (range 200))+ (set (map #(get-in % [:payload :data "dashboard_id"]) @received)))))))))