Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Metaplow Analytics for Internal Metabase Instance (#72380)

Ryan Laurie4c551c71fb06
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index d1a37f28fdc..5fbd9803eab 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -187,9 +187,9 @@
{:team "UX West"
:api #{metabase.analytics.api
metabase.analytics.core
+ metabase.analytics.event
metabase.analytics.init
- metabase.analytics.settings
- metabase.analytics.snowplow}
+ metabase.analytics.settings}
:uses #{analytics-interface
api
app-db
diff --git a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj b/.clj-kondo/src/hooks/metabase/settings/models/setting.clj
index d31472af446..d7d5d1e5253 100644
--- a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj
+++ b/.clj-kondo/src/hooks/metabase/settings/models/setting.clj
@@ -95,6 +95,8 @@
metabot-feedback-url
metabot-get-prompt-templates-url
metabot-prompt-generator-token-limit
+ metaplow-tracking-enabled
+ metaplow-url
multi-setting-read-only
notebook-native-preview-sidebar-width
notification-link-base-url
diff --git a/enterprise/backend/src/metabase_enterprise/gsheets/api.clj b/enterprise/backend/src/metabase_enterprise/gsheets/api.clj
index 85db9373008..52ae0d206d4 100644
--- a/enterprise/backend/src/metabase_enterprise/gsheets/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/gsheets/api.clj
@@ -8,7 +8,7 @@
:refer [gsheets gsheets!]]
[metabase-enterprise.harbormaster.client :as hm.client]
[metabase.analytics-interface.core :as analytics]
- [metabase.analytics.snowplow :as snowplow]
+ [metabase.analytics.event :as analytics.event]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.util :as u]
@@ -229,7 +229,7 @@
[{} {} {:keys [url]} :- [:map [:url ms/NonBlankString]]]
(let [attached-dwh (t2/select-one-fn :id :model/Database :is_attached_dwh true)]
(when-not (some? attached-dwh)
- (snowplow/track-event! :snowplow/simple_event {:event "sheets_connected" :event_detail "fail - no dwh"})
+ (analytics.event/track-event! :snowplow/simple_event {:event "sheets_connected" :event_detail "fail - no dwh"})
(throw-error 400 (tru "No attached dwh found.") nil))
(let [[status response] (hm-create-gdrive-conn! url)
@@ -359,10 +359,10 @@
(let [sheet-config (gsheets-safe)]
(if (empty? sheet-config)
(do
- (snowplow/track-event! :snowplow/simple_event {:event "sheets_sync" :event_detail "fail - no config"})
+ (analytics.event/track-event! :snowplow/simple_event {:event "sheets_sync" :event_detail "fail - no config"})
(throw-error 404 (tru "No attached google sheet(s) found.") nil))
(do
- (snowplow/track-event! :snowplow/simple_event {:event "sheets_sync"})
+ (analytics.event/track-event! :snowplow/simple_event {:event "sheets_sync"})
(analytics/inc! :metabase-gsheets/connection-manually-synced)
(let [[status response] (hm-sync-conn! (:gdrive/conn-id sheet-config))]
(if (= status :ok)
@@ -378,7 +378,7 @@
(api.macros/defendpoint :delete "/connection"
"Disconnect the google service account. There is only one (or zero) at the time of writing."
[]
- (snowplow/track-event! :snowplow/simple_event {:event "sheets_disconnected"})
+ (analytics.event/track-event! :snowplow/simple_event {:event "sheets_disconnected"})
(analytics/inc! :metabase-gsheets/connection-deleted)
(reset-gsheets-status!)
{:status "not-connected"})
diff --git a/enterprise/backend/src/metabase_enterprise/security_center/models/security_advisory.clj b/enterprise/backend/src/metabase_enterprise/security_center/models/security_advisory.clj
index a7ec4348d58..3a998abd2b5 100644
--- a/enterprise/backend/src/metabase_enterprise/security_center/models/security_advisory.clj
+++ b/enterprise/backend/src/metabase_enterprise/security_center/models/security_advisory.clj
@@ -1,6 +1,6 @@
(ns metabase-enterprise.security-center.models.security-advisory
(:require
- [metabase.analytics.snowplow :as snowplow]
+ [metabase.analytics.core :as analytics]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
[methodical.core :as methodical]
@@ -49,9 +49,9 @@
(events/publish-event! :event/security-advisory-acknowledge
{:object advisory
:user-id user-id})
- (snowplow/track-event! :snowplow/simple_event
- {:event "security_advisory_acknowledged"
- :event_detail (name (:severity advisory))})
+ (analytics/track-event! :snowplow/simple_event
+ {:event "security_advisory_acknowledged"
+ :event_detail (name (:severity advisory))})
(-> (t2/select-one :model/SecurityAdvisory :id (:id advisory))
(t2/hydrate :acknowledged_by_user))))
diff --git a/enterprise/backend/src/metabase_enterprise/security_center/notification.clj b/enterprise/backend/src/metabase_enterprise/security_center/notification.clj
index d5b68ff159c..088c8944f64 100644
--- a/enterprise/backend/src/metabase_enterprise/security_center/notification.clj
+++ b/enterprise/backend/src/metabase_enterprise/security_center/notification.clj
@@ -10,7 +10,7 @@
included as a recipient when set."
(:require
[metabase-enterprise.security-center.settings :as settings]
- [metabase.analytics.snowplow :as snowplow]
+ [metabase.analytics.core :as analytics]
[metabase.channel.settings :as channel.settings]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
@@ -88,11 +88,11 @@
"Track a Snowplow event for each channel in the notification's handlers."
[notification triggered-from result]
(doseq [{:keys [channel_type]} (:handlers notification)]
- (snowplow/track-event! :snowplow/simple_event
- {:event "security_advisory_notification_sent"
- :event_detail (channel-type->name channel_type)
- :triggered_from triggered-from
- :result result})))
+ (analytics/track-event! :snowplow/simple_event
+ {:event "security_advisory_notification_sent"
+ :event_detail (channel-type->name channel_type)
+ :triggered_from triggered-from
+ :result result})))
(def ^:private test-advisory
"A synthetic advisory used for test notifications so admins can verify delivery."
diff --git a/frontend/src/metabase-types/api/mocks/settings.ts b/frontend/src/metabase-types/api/mocks/settings.ts
index 00d721a0be6..88a8e7332ad 100644
--- a/frontend/src/metabase-types/api/mocks/settings.ts
+++ b/frontend/src/metabase-types/api/mocks/settings.ts
@@ -175,6 +175,8 @@ export const createMockSettings = (
"custom-viz-plugin-dev-mode-enabled": false,
"allowed-iframe-hosts": "*",
"anon-tracking-enabled": false,
+ "metaplow-tracking-enabled": false,
+ "metaplow-url": null,
"application-colors": {},
"application-font": "Lato",
"application-font-files": [],
diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.ts
index 74afea6c26c..2d2a118ed32 100644
--- a/frontend/src/metabase-types/api/settings.ts
+++ b/frontend/src/metabase-types/api/settings.ts
@@ -551,6 +551,8 @@ interface PublicSettings {
"agent-api-enabled?": boolean;
"analytics-uuid": string;
"anon-tracking-enabled": boolean;
+ "metaplow-tracking-enabled": boolean;
+ "metaplow-url": string | null;
"application-font": string;
"application-font-files": FontFile[] | null;
"application-name": string;
diff --git a/frontend/src/metabase/analytics/event.ts b/frontend/src/metabase/analytics/event.ts
index 65da66f6915..98aeb23756d 100644
--- a/frontend/src/metabase/analytics/event.ts
+++ b/frontend/src/metabase/analytics/event.ts
@@ -1,6 +1,7 @@
import * as Snowplow from "@snowplow/browser-tracker";
import { shouldLogAnalytics } from "metabase/env";
+import { trackMetaplowEvent } from "metabase/utils/metaplow";
import Settings from "metabase/utils/settings";
import type { SchemaEventMap, SchemaType } from "metabase-types/analytics";
import type { SimpleEventSchema } from "metabase-types/analytics/event";
@@ -43,14 +44,14 @@ export function trackSchemaEvent<S extends SchemaType>(
schema: S,
event: SchemaEventMap[S],
): void {
- const shouldSendEvent =
- Settings.trackingEnabled() && Settings.snowplowEnabled();
+ const trackingEnabled = Settings.trackingEnabled();
+ const shouldSendSnowplow = Settings.snowplowEnabled();
if (shouldLogAnalytics) {
const { event: type, ...other } = event;
// eslint-disable-next-line no-console
console.log(
- `%c[SNOWPLOW EVENT | event sent:${shouldSendEvent}]%c, ${type}`,
+ `%c[SNOWPLOW EVENT | event sent:${shouldSendSnowplow}]%c, ${type}`,
// eslint-disable-next-line metabase/no-color-literals
"background: #222; color: #bada55",
"color: ",
@@ -58,7 +59,11 @@ export function trackSchemaEvent<S extends SchemaType>(
);
}
- if (shouldSendEvent) {
+ if (!trackingEnabled) {
+ return;
+ }
+
+ if (shouldSendSnowplow) {
Snowplow.trackSelfDescribingEvent({
event: {
schema: `iglu:com.metabase/${schema}/jsonschema/${VERSIONS[schema]}`,
@@ -66,4 +71,9 @@ export function trackSchemaEvent<S extends SchemaType>(
},
});
}
+
+ if (Settings.get("metaplow-tracking-enabled")) {
+ const { event: name, ...data } = event;
+ trackMetaplowEvent(name, data);
+ }
}
diff --git a/frontend/src/metabase/analytics/page-view.ts b/frontend/src/metabase/analytics/page-view.ts
index bc0a50d8ef2..ee856673bc6 100644
--- a/frontend/src/metabase/analytics/page-view.ts
+++ b/frontend/src/metabase/analytics/page-view.ts
@@ -1,15 +1,22 @@
import * as Snowplow from "@snowplow/browser-tracker";
+import { trackMetaplowPageView } from "metabase/utils/metaplow";
import Settings from "metabase/utils/settings";
export const trackPageView = (url: string): void => {
- if (!url || !Settings.trackingEnabled() || !Settings.snowplowEnabled()) {
+ if (!url || !Settings.trackingEnabled()) {
return;
}
- Snowplow.setReferrerUrl("#");
- Snowplow.setCustomUrl(getSanitizedUrl(url));
- Snowplow.trackPageView();
+ if (Settings.snowplowEnabled()) {
+ Snowplow.setReferrerUrl("#");
+ Snowplow.setCustomUrl(getSanitizedUrl(url));
+ Snowplow.trackPageView();
+ }
+
+ if (Settings.get("metaplow-tracking-enabled")) {
+ trackMetaplowPageView(url);
+ }
};
const getSanitizedUrl = (url: string): string => {
diff --git a/frontend/src/metabase/utils/metaplow.ts b/frontend/src/metabase/utils/metaplow.ts
new file mode 100644
index 00000000000..eac8cec26d2
--- /dev/null
+++ b/frontend/src/metabase/utils/metaplow.ts
@@ -0,0 +1,77 @@
+import Settings from "metabase/utils/settings";
+
+const METAPLOW_WEBSITE_ID = "23eefa30-4c4f-490e-aa4f-084cd23b1561";
+const anonymizedHostname = "anonymous.metabase.com";
+const anonymizedOrigin = `http://${anonymizedHostname}`;
+
+type MetaplowPayload = {
+ website: string;
+ url: string;
+ id: string;
+ referrer: string;
+ title: string;
+ hostname: string;
+ screen: string;
+ language: string;
+ tag: string;
+ name: string;
+ data?: Record<string, unknown>;
+};
+
+const getSanitizedUrl = (url: string) => {
+ const parsed = new URL(url, window.location.origin);
+ const pathWithoutSlug = parsed.pathname.replace(
+ /(\/\d+)-[^\/]+$/,
+ (_, path) => path,
+ );
+ return anonymizedOrigin + pathWithoutSlug;
+};
+
+function getBasePayload(url: string): Omit<MetaplowPayload, "name" | "data"> {
+ return {
+ website: METAPLOW_WEBSITE_ID,
+ url: getSanitizedUrl(url),
+ id: Settings.get("analytics-uuid") ?? "",
+ referrer: "",
+ title: "",
+ hostname: anonymizedHostname,
+ screen: `${window.screen.width}x${window.screen.height}`,
+ language: navigator.language,
+ tag: "metabase-instance",
+ };
+}
+
+async function send(payload: MetaplowPayload): Promise<unknown> {
+ const metaplowUrl = Settings.get("metaplow-url");
+ if (!metaplowUrl) {
+ return;
+ }
+
+ const headers: Record<string, string> = {
+ "Content-Type": "application/json",
+ };
+
+ return fetch(metaplowUrl, {
+ method: "POST",
+ headers,
+ body: JSON.stringify({ type: "event", payload }),
+ });
+}
+
+export function trackMetaplowEvent(
+ name: MetaplowPayload["name"],
+ data: MetaplowPayload["data"] = {},
+): void {
+ send({
+ ...getBasePayload(window.location.href),
+ name,
+ data,
+ }).catch(() => undefined);
+}
+
+export function trackMetaplowPageView(url: string): Promise<unknown> {
+ return send({
+ ...getBasePayload(url),
+ name: "pageview",
+ }).catch(() => undefined);
+}
diff --git a/frontend/src/metabase/utils/metaplow.unit.spec.ts b/frontend/src/metabase/utils/metaplow.unit.spec.ts
new file mode 100644
index 00000000000..4ec2b562745
--- /dev/null
+++ b/frontend/src/metabase/utils/metaplow.unit.spec.ts
@@ -0,0 +1,123 @@
+import Settings from "metabase/utils/settings";
+
+import { trackMetaplowEvent, trackMetaplowPageView } from "./metaplow";
+
+const METAPLOW_URL = "https://metaplow.example.com/api/send";
+const METAPLOW_WEBSITE_ID = "23eefa30-4c4f-490e-aa4f-084cd23b1561";
+const ANON_ORIGIN = "http://anonymous.metabase.com";
+
+describe("metaplow", () => {
+ let fetchSpy: jest.SpyInstance;
+
+ beforeEach(() => {
+ Settings.set("metaplow-url", METAPLOW_URL);
+ fetchSpy = jest
+ .spyOn(global, "fetch")
+ .mockResolvedValue(new Response(null, { status: 200 }));
+ });
+
+ afterEach(() => {
+ fetchSpy.mockRestore();
+ Settings.set("metaplow-url", null);
+ });
+
+ const getSentPayload = () => {
+ expect(fetchSpy).toHaveBeenCalledTimes(1);
+ const [url, init] = fetchSpy.mock.calls[0];
+ expect(url).toBe(METAPLOW_URL);
+ expect(init?.method).toBe("POST");
+ expect((init?.headers as Record<string, string>)["Content-Type"]).toBe(
+ "application/json",
+ );
+ return JSON.parse(init?.body as string);
+ };
+
+ describe("trackMetaplowEvent", () => {
+ it("does not call fetch when metaplow-url is not set", () => {
+ Settings.set("metaplow-url", null);
+ trackMetaplowEvent("button_clicked");
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
+
+ it("sends an event with the given name and default empty data", () => {
+ trackMetaplowEvent("button_clicked");
+
+ const body = getSentPayload();
+ expect(body.type).toBe("event");
+ expect(body.payload).toEqual(
+ expect.objectContaining({
+ website: METAPLOW_WEBSITE_ID,
+ name: "button_clicked",
+ data: {},
+ referrer: "",
+ title: "",
+ hostname: "anonymous.metabase.com",
+ tag: "metabase-instance",
+ }),
+ );
+ });
+
+ it("forwards the data payload", () => {
+ trackMetaplowEvent("button_clicked", { foo: "bar", count: 3 });
+
+ const { payload } = getSentPayload();
+ expect(payload.data).toEqual({ foo: "bar", count: 3 });
+ });
+
+ it("includes screen and language from the browser", () => {
+ trackMetaplowEvent("button_clicked");
+
+ const { payload } = getSentPayload();
+ expect(payload.screen).toBe(
+ `${window.screen.width}x${window.screen.height}`,
+ );
+ expect(payload.language).toBe(navigator.language);
+ });
+
+ it("uses an anonymized hostname in the url", () => {
+ trackMetaplowEvent("button_clicked");
+
+ const { payload } = getSentPayload();
+ expect(payload.url.startsWith(ANON_ORIGIN)).toBe(true);
+ });
+ });
+
+ describe("trackMetaplowPageView", () => {
+ it("does not call fetch when metaplow-url is not set", async () => {
+ Settings.set("metaplow-url", null);
+ await trackMetaplowPageView("/question/42-my-question");
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
+
+ it('sends an event with name "pageview"', async () => {
+ await trackMetaplowPageView("/dashboard/1");
+
+ const { payload } = getSentPayload();
+ expect(payload.name).toBe("pageview");
+ expect(payload.url).toBe(`${ANON_ORIGIN}/dashboard/1`);
+ });
+
+ it("strips slugs from /:id-slug paths", async () => {
+ await trackMetaplowPageView("/question/42-my-favorite-question");
+
+ const { payload } = getSentPayload();
+ expect(payload.url).toBe(`${ANON_ORIGIN}/question/42`);
+ });
+
+ it("anonymizes absolute URLs by replacing the origin", async () => {
+ await trackMetaplowPageView(
+ "https://my-company.metabaseapp.com/collection/5-secrets",
+ );
+
+ const { payload } = getSentPayload();
+ expect(payload.url).toBe(`${ANON_ORIGIN}/collection/5`);
+ });
+
+ it("preserves paths without a numeric slug prefix", async () => {
+ await trackMetaplowPageView("/admin/settings/general");
+
+ const { payload } = getSentPayload();
+ expect(payload.url).toBe(`${ANON_ORIGIN}/admin/settings/general`);
+ });
+ });
+});
diff --git a/src/metabase/analytics/core.clj b/src/metabase/analytics/core.clj
index 05f118644eb..56dce821119 100644
--- a/src/metabase/analytics/core.clj
+++ b/src/metabase/analytics/core.clj
@@ -1,22 +1,22 @@
(ns metabase.analytics.core
(:require
+ [metabase.analytics.event]
[metabase.analytics.llm-token-usage]
[metabase.analytics.prometheus]
[metabase.analytics.quartz]
[metabase.analytics.sdk]
[metabase.analytics.settings]
- [metabase.analytics.snowplow]
[metabase.analytics.stats]
[metabase.analytics.util]
[potemkin :as p]))
(comment
+ metabase.analytics.event/keep-me
metabase.analytics.llm-token-usage/keep-me
metabase.analytics.prometheus/keep-me
metabase.analytics.quartz/keep-me
metabase.analytics.sdk/keep-me
metabase.analytics.settings/keep-me
- metabase.analytics.snowplow/keep-me
metabase.analytics.stats/keep-me
metabase.analytics.util/keep-me)
@@ -65,7 +65,7 @@
anon-tracking-enabled!
instance-creation]
- [metabase.analytics.snowplow
+ [metabase.analytics.event
track-event!]
diff --git a/src/metabase/analytics/event.clj b/src/metabase/analytics/event.clj
new file mode 100644
index 00000000000..e5af10a60e2
--- /dev/null
+++ b/src/metabase/analytics/event.clj
@@ -0,0 +1,17 @@
+(ns metabase.analytics.event
+ "Dispatches analytics events to Snowplow and Metaplow. Each tracker gates itself on its own setting; this namespace
+ just fans the call out."
+ (:require
+ [metabase.analytics.metaplow :as metaplow]
+ [metabase.analytics.snowplow :as snowplow]
+ [metabase.api.common :as api]))
+
+(defn track-event!
+ "Send a single analytics event to Snowplow and Metaplow. Each tracker decides whether to emit based on its own
+ setting."
+ ([schema data]
+ (track-event! schema data api/*current-user-id*))
+ ([schema data user-id]
+ (let [snowplow? (snowplow/track-event! schema data user-id)
+ metaplow? (metaplow/track-event! schema data user-id)]
+ (or snowplow? metaplow?))))
diff --git a/src/metabase/analytics/llm_token_usage.clj b/src/metabase/analytics/llm_token_usage.clj
index d062fb169ac..74eb619faf0 100644
--- a/src/metabase/analytics/llm_token_usage.clj
+++ b/src/metabase/analytics/llm_token_usage.clj
@@ -2,7 +2,7 @@
"LLM token usage tracking for Snowplow and Prometheus."
(:require
[metabase.analytics-interface.core :as analytics]
- [metabase.analytics.snowplow :as snowplow]
+ [metabase.analytics.event :as analytics.event]
[metabase.analytics.util :as analytics.util]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]))
@@ -34,24 +34,24 @@
estimated-costs-usd user-id duration-ms source tag session-id profile
hashed-metabase-license-token]}
:- SnowplowArgs]
- (snowplow/track-event! :snowplow/token_usage
- {:hashed-metabase-license-token (or hashed-metabase-license-token
- (analytics.util/hashed-metabase-token-or-uuid))
- :request-id request-id
- :model-id model-id
- :total-tokens total-tokens
- :prompt-tokens prompt-tokens
- :completion-tokens completion-tokens
- :cache-creation-tokens cache-creation-tokens
- :cache-read-tokens cache-read-tokens
- :estimated-costs-usd estimated-costs-usd
- :user-id user-id
- :duration-ms (some-> duration-ms long)
- :source source
- :tag tag
- :session-id session-id
- :profile profile}
- user-id))
+ (analytics.event/track-event! :snowplow/token_usage
+ {:hashed-metabase-license-token (or hashed-metabase-license-token
+ (analytics.util/hashed-metabase-token-or-uuid))
+ :request-id request-id
+ :model-id model-id
+ :total-tokens total-tokens
+ :prompt-tokens prompt-tokens
+ :completion-tokens completion-tokens
+ :cache-creation-tokens cache-creation-tokens
+ :cache-read-tokens cache-read-tokens
+ :estimated-costs-usd estimated-costs-usd
+ :user-id user-id
+ :duration-ms (some-> duration-ms long)
+ :source source
+ :tag tag
+ :session-id session-id
+ :profile profile}
+ user-id))
(def ^:private PrometheusArgs
[:map
diff --git a/src/metabase/analytics/metaplow.clj b/src/metabase/analytics/metaplow.clj
new file mode 100644
index 00000000000..8816b0710da
--- /dev/null
+++ b/src/metabase/analytics/metaplow.clj
@@ -0,0 +1,201 @@
+(ns metabase.analytics.metaplow
+ "Sends analytics events to a Metaplow (Umami-compatible) collector.
+
+ `track-event!` builds a payload, offers it onto an async send queue, and returns. A `pipeline-blocking` worker pool
+ drains the queue and POSTs each event to the collector's `/api/send` endpoint, retrying transient failures via
+ [[metabase.util.retry/with-retry]]. The HTTP connection pool is sized to match the worker count so workers never
+ contend for slots.
+
+ Emission is gated by [[metabase.analytics.settings/metaplow-tracking-enabled]]. The public entry point used by the
+ rest of the codebase is [[metabase.analytics.event/track-event!]], which fans out to both Snowplow and Metaplow."
+ (:require
+ [clj-http.client :as http]
+ [clj-http.conn-mgr :as conn-mgr]
+ [clojure.core.async :as a]
+ [clojure.walk :as walk]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.analytics.settings :as analytics.settings]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.util :as u]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.retry :as retry]
+ [metabase.version.core :as version]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private website-id
+ "Same constant the frontend tracker uses; see frontend/src/metabase/utils/metaplow.ts."
+ "23eefa30-4c4f-490e-aa4f-084cd23b1561")
+
+(def ^:private anonymized-hostname
+ "Hostname sent on every event. Matches the FE constant in frontend/src/metabase/utils/metaplow.ts."
+ "anonymous.metabase.com")
+
+(def ^:private payload-tag
+ "Tag identifying the event source. Matches the FE constant in frontend/src/metabase/utils/metaplow.ts."
+ "metabase-instance")
+
+(def ^:private queue-size 10000)
+(def ^:private no-retry-status-codes #{400 401 403 410 422})
+
+;; `worker-count` drives both the number of worker threads draining the queue AND the size of the HTTP connection
+;; pool. Keep them coupled: if the pool is smaller than the worker count, workers contend for connection slots and
+;; the parallelism is silently capped at the pool size.
+(def ^:private worker-count 50)
+
+(defn- normalize-kw
+ [kw]
+ (name (u/->snake_case_en kw)))
+
+(defn- snake-case-payload
+ "Convert any keyword in a payload (keys or values, at any depth) to a snake_case string."
+ [m]
+ (walk/postwalk #(if (keyword? %) (normalize-kw %) %) m))
+
+(defn- event-name+data
+ "Returns `[event-name event-data]` for a given Snowplow-style schema kw and payload. Schemas with an `:event`
+ discriminator become `\"<schema>.<event>\"`; schemas without one become `\"<schema>\"`."
+ [schema data]
+ (let [schema-name (normalize-kw schema)
+ data* (snake-case-payload data)]
+ (if-let [event (get data* "event")]
+ [(str schema-name "." event) (dissoc data* "event")]
+ [schema-name data*])))
+
+(defn- build-payload
+ [schema data]
+ (let [[event-name event-data] (event-name+data schema data)
+ enriched-data (assoc event-data
+ "version_tag" (:tag (version/version))
+ "plan" (or (premium-features/plan-alias) "oss"))]
+ {:type "event"
+ :payload {:website website-id
+ :id (analytics.settings/analytics-uuid)
+ :hostname anonymized-hostname
+ :tag payload-tag
+ :name event-name
+ :data enriched-data}}))
+
+(defonce ^:private
+ ^{:doc "Reuses TCP/TLS connections across requests to the metaplow collector. Same idea as Snowplow's
+ `PoolingHttpClientConnectionManager` setup in `metabase.analytics.snowplow`, just via clj-http's wrapper.
+ Sized to `worker-count`: every request goes to one host, so `:default-per-route` is the actual parallelism
+ cap. Wrapped in `delay` so the pool isn't created until the first event is sent."}
+ connection-manager
+ (delay (conn-mgr/make-reusable-conn-manager {:threads worker-count
+ :default-per-route worker-count})))
+
+(defn- send-event!
+ "POST a single payload to the Metaplow `/api/send` endpoint. Returns a map with `:status` (HTTP status code, or -1
+ on connection failure)."
+ [payload]
+ (try
+ (http/post (analytics.settings/metaplow-url)
+ {:body (json/encode payload)
+ :content-type :json
+ :socket-timeout 5000
+ :connection-timeout 5000
+ :throw-exceptions false
+ :connection-manager @connection-manager})
+ (catch Throwable e
+ (analytics/inc! :metabase-metaplow/errors {:stage :send-event!})
+ (log/warn e "Connection failure sending Metaplow event")
+ {:status -1})))
+
+(defn- retryable-response?
+ "Truthy when a `send-event!` response should trigger another attempt: non-2xx and not one of the no-retry
+ status codes that Snowplow also gives up on (400/401/403/410/422)."
+ [{:keys [status]}]
+ (and (some? status)
+ (not (<= 200 status 299))
+ (not (contains? no-retry-status-codes status))))
+
+(defn- send-event-with-retries!
+ "Send an event, retrying with exponential backoff on connection failures or retryable HTTP statuses (via
+ [[metabase.util.retry/with-retry]]). Returns `:sent` on success and `:error` on failure (never nil), so callers can
+ reliably distinguish outcomes."
+ [payload]
+ (try
+ (retry/with-retry {:retry-if (fn [response _ex] (retryable-response? response))}
+ (send-event! payload))
+ :sent
+ (catch Throwable e
+ (analytics/inc! :metabase-metaplow/errors {:stage :send-event-with-retries!})
+ (log/warn e "Error sending Metaplow event")
+ :error)))
+
+(defonce ^:private
+ ^{:doc "Individual payloads queued by `track-event!` land here. Bounded chan of size `queue-size`; new events are
+ rejected when full."}
+ source-chan
+ (a/chan queue-size))
+
+(defonce ^:private
+ ^{:doc "Black-hole sink for `pipeline-blocking` worker output. Sliding-buffer with no consumer, so workers' puts
+ never block."}
+ sink-chan
+ (a/chan (a/sliding-buffer 1)))
+
+(defonce ^:private
+ ^{:doc "Spawns `worker-count` worker threads that drain `source-chan` and apply `send-event-with-retries!` to each
+ event. Wrapped in `delay` so workers aren't spawned until the first event is enqueued — instances that don't
+ use Metaplow pay nothing at boot."}
+ pipeline
+ (delay
+ (a/pipeline-blocking
+ worker-count
+ sink-chan
+ ;; The var (not the fn) is passed so the indirection survives `with-redefs` in tests.
+ (map #'send-event-with-retries!)
+ source-chan)))
+
+(defn- enqueue!
+ "Push a payload onto the async send queue. Returns true on success, false when the queue is full."
+ [payload]
+ @pipeline ;; start the pipeline on first call
+ (or (a/offer! source-chan payload)
+ (do (analytics/inc! :metabase-metaplow/errors {:stage :enqueue!})
+ false)))
+
+(mu/defn track-event! :- :boolean
+ "Send a single analytics event to the Metaplow collector. Returns true when the event was enqueued, false when
+ Metaplow tracking is disabled or the queue is full."
+ ([schema data]
+ (track-event! schema data nil))
+ ([schema :- :keyword data _user-id]
+ (boolean
+ (when (analytics.settings/metaplow-tracking-enabled)
+ (try
+ (enqueue! (build-payload schema data))
+ (catch Throwable e
+ (analytics/inc! :metabase-metaplow/errors {:stage :track-event!})
+ (log/warn e "Error queueing Metaplow event")
+ false))))))
+
+(comment
+ ;; Manual testing against a real Umami/Metaplow collector. Eval forms in the REPL one at a time.
+
+ ;; 1. Point at the collector.
+ (analytics.settings/metaplow-url! "https://product-analytics-ingestion.staging.metabase.com/api/send")
+ (analytics.settings/anon-tracking-enabled! true)
+
+ ;; 2. Sanity check.
+ (analytics.settings/metaplow-tracking-enabled) ; => true
+
+ ;; 3. Fire one event: queued, drained by a worker, POSTed to `metaplow-url`.
+ (track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id 1})
+
+ ;; 4. Fire a burst: workers POST in parallel up to `worker-count`.
+ (dotimes [i 75]
+ (track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id i}))
+
+ ;; 5. Inspect what gets built without sending anything.
+ (build-payload :snowplow/dashboard {:event :dashboard-created :dashboard-id 42})
+
+ ;; 6. POST a hand-crafted event directly. Bypasses the queue; useful for testing URL, auth, and payload shape.
+ (send-event! (build-payload :snowplow/dashboard {:event :test-event :dashboard-id 1}))
+
+ ;; 7. Turn it off.
+ (analytics.settings/metaplow-url! nil))
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index bec9452d237..6bfa7255f13 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -690,7 +690,12 @@
{:description "Unix timestamp (seconds since epoch) of the last successful Security Center."})
(prometheus/gauge :metabase-security-center/vulnerable-advisories
{:description "Number of advisories where this Metabase instance is potentially affected."
- :labels [:severity :acknowledged]})])
+ :labels [:severity :acknowledged]})
+
+ ;; metaplow analytics metrics
+ (prometheus/counter :metabase-metaplow/errors
+ {:description "Metaplow event pipeline errors by stage."
+ :labels [:stage]})])
(defn- quartz-collectors
[]
diff --git a/src/metabase/analytics/settings.clj b/src/metabase/analytics/settings.clj
index 1b7713f8732..d14be069769 100644
--- a/src/metabase/analytics/settings.clj
+++ b/src/metabase/analytics/settings.clj
@@ -44,6 +44,23 @@
:visibility :public
:doc false)
+(defsetting metaplow-tracking-enabled
+ (deferred-tru
+ (str "Boolean indicating whether analytics events are being sent to Metaplow. "
+ "True if anonymous tracking is enabled for this instance, and a Metaplow collector URL is set."))
+ :type :boolean
+ :getter (fn [] (boolean (and (anon-tracking-enabled)
+ (setting/get-value-of-type :string :metaplow-url))))
+ :visibility :public
+ :doc false)
+
+(defsetting metaplow-url
+ (deferred-tru "The URL of the Metaplow collector to send analytics events to.")
+ :encryption :no
+ :visibility :public
+ :audit :never
+ :doc false)
+
(defsetting snowplow-url
(deferred-tru "The URL of the Snowplow collector to send analytics events to.")
:encryption :no
@@ -67,7 +84,7 @@
;; is first read.
(let [value (or (first-user-creation) (t/offset-date-time))]
(setting/set-value-of-type! :timestamp :instance-creation value)
- ((requiring-resolve 'metabase.analytics.snowplow/track-event!) :snowplow/account {:event :new_instance_created} nil)))
+ ((requiring-resolve 'metabase.analytics.event/track-event!) :snowplow/account {:event :new_instance_created} nil)))
(u.date/format-rfc3339 (setting/get-value-of-type :timestamp :instance-creation)))
(defsetting instance-creation
diff --git a/src/metabase/analytics/stats.clj b/src/metabase/analytics/stats.clj
index 8ed4caed580..da701d63329 100644
--- a/src/metabase/analytics/stats.clj
+++ b/src/metabase/analytics/stats.clj
@@ -9,8 +9,8 @@
[environ.core :as env]
[java-time.api :as t]
[medley.core :as m]
+ [metabase.analytics.event :as analytics.event]
[metabase.analytics.settings :as analytics.settings]
- [metabase.analytics.snowplow :as snowplow]
[metabase.app-db.core :as app-db]
[metabase.appearance.core :as appearance]
[metabase.config.core :as config]
@@ -1048,5 +1048,5 @@
(str "Missing required keys in snowplow-data. got:" (sort (keys snowplow-data))))
#_{:clj-kondo/ignore [:deprecated-var]}
(send-stats-deprecated! stats)
- (snowplow/track-event! :snowplow/instance_stats snowplow-data)
+ (analytics.event/track-event! :snowplow/instance_stats snowplow-data)
(stats-post-cleanup))))
diff --git a/src/metabase/llm/api.clj b/src/metabase/llm/api.clj
index 645b04a53fd..2364a5dc341 100644
--- a/src/metabase/llm/api.clj
+++ b/src/metabase/llm/api.clj
@@ -4,7 +4,6 @@
[clojure.java.io :as io]
[clojure.set :as set]
[metabase.analytics.core :as analytics]
- [metabase.analytics.snowplow :as snowplow]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
@@ -80,12 +79,12 @@
(defn- track-sqlgen-event!
"Track SQL generation usage via Snowplow simple_event."
[{:keys [duration-ms result engine]}]
- (snowplow/track-event! :snowplow/simple_event
- {:event "metabot_oss_sqlgen_used"
- :event_detail (some-> engine name)
- :duration_ms (some-> duration-ms long)
- :result result}
- api/*current-user-id*))
+ (analytics/track-event! :snowplow/simple_event
+ {:event "metabot_oss_sqlgen_used"
+ :event_detail (some-> engine name)
+ :duration_ms (some-> duration-ms long)
+ :result result}
+ api/*current-user-id*))
(api.macros/defendpoint :get "/list-models"
:- [:map [:models [:sequential [:map
diff --git a/src/metabase/logger/api.clj b/src/metabase/logger/api.clj
index eebb13f053e..cb556f7ecb7 100644
--- a/src/metabase/logger/api.clj
+++ b/src/metabase/logger/api.clj
@@ -3,7 +3,7 @@
(:require
[clojure.string :as str]
[flatland.ordered.map :as ordered-map]
- [metabase.analytics.snowplow :as snowplow]
+ [metabase.analytics.core :as analytics]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.logger.core :as logger]
@@ -224,10 +224,10 @@
(when-let [task @log-adjustment]
(cancel-undo-task! task))
(let [plan (do (if (empty? log-levels)
- (snowplow/track-event! :snowplow/simple_event {:event "log_adjustments_reset"})
- (snowplow/track-event! :snowplow/simple_event {:event "log_adjustments_set"
- :event_detail (->seconds-str duration_unit
- duration)}))
+ (analytics/track-event! :snowplow/simple_event {:event "log_adjustments_reset"})
+ (analytics/track-event! :snowplow/simple_event {:event "log_adjustments_set"
+ :event_detail (->seconds-str duration_unit
+ duration)}))
(set-log-levels! (update-vals log-levels keyword)))]
(reset! log-adjustment {:plan plan, :undo-task (undo-task plan duration duration_unit)})))
nil)
@@ -241,7 +241,7 @@
[]
(api/check-superuser)
(when-let [task @log-adjustment]
- (snowplow/track-event! :snowplow/simple_event {:event "log_adjustments_reset"})
+ (analytics/track-event! :snowplow/simple_event {:event "log_adjustments_reset"})
(cancel-undo-task! task)
(reset! log-adjustment nil))
nil)
diff --git a/src/metabase/login_history/record.clj b/src/metabase/login_history/record.clj
index b646beed511..0cfabd40859 100644
--- a/src/metabase/login_history/record.clj
+++ b/src/metabase/login_history/record.clj
@@ -1,6 +1,6 @@
(ns metabase.login-history.record
(:require
- [metabase.analytics.snowplow :as snowplow]
+ [metabase.analytics.core :as analytics]
[metabase.channel.email.messages :as messages]
[metabase.login-history.models.login-history :as login-history]
[metabase.login-history.settings :as login-history.settings]
@@ -41,4 +41,4 @@
(when-not (or (:embedded device-info) (:token_exchange device-info))
(maybe-send-login-from-new-device-email history-entry))
(when-not (:last_login user)
- (snowplow/track-event! :snowplow/account {:event :new-user-created} (u/the-id user)))))
+ (analytics/track-event! :snowplow/account {:event :new-user-created} (u/the-id user)))))
diff --git a/src/metabase/server/middleware/security.clj b/src/metabase/server/middleware/security.clj
index c89fb369207..236fe3b8932 100644
--- a/src/metabase/server/middleware/security.clj
+++ b/src/metabase/server/middleware/security.clj
@@ -187,6 +187,8 @@
;; Snowplow analytics
(when (analytics/anon-tracking-enabled)
(setting/get-value-of-type :string :snowplow-url))
+ (when (analytics/anon-tracking-enabled)
+ (setting/get-value-of-type :string :metaplow-url))
;; Webpack dev server
(when config/is-dev?
(str "*:" frontend-dev-port " ws://*:" frontend-dev-port))
diff --git a/test/metabase/analytics/metaplow_test.clj b/test/metabase/analytics/metaplow_test.clj
new file mode 100644
index 00000000000..ac392ca1d5a
--- /dev/null
+++ b/test/metabase/analytics/metaplow_test.clj
@@ -0,0 +1,104 @@
+(ns metabase.analytics.metaplow-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.analytics.metaplow :as metaplow]
+ [metabase.analytics.settings :as analytics.settings]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [metabase.version.core :as version])
+ (:import
+ (java.util.concurrent CountDownLatch TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(deftest build-payload-test
+ (testing "Schemas with an `:event` key produce `<schema>.<event>` and the event key is removed from data"
+ (let [payload (#'metaplow/build-payload :snowplow/dashboard
+ {:event :dashboard-created :dashboard-id 42 :num-tabs 1})]
+ (is (=? {:type "event"
+ :payload {:name "dashboard.dashboard_created"
+ :data {"dashboard_id" 42
+ "num_tabs" 1}}}
+ payload))
+ (is (not (contains? (get-in payload [:payload :data]) "event")))))
+
+ (testing "Schemas without an `:event` key produce `<schema>` and keep the full payload"
+ (is (=? {:payload {:name "instance_stats"
+ :data {"metric_one" 1
+ "metric_two" 2}}}
+ (#'metaplow/build-payload :snowplow/instance_stats {:metric_one 1 :metric_two 2}))))
+
+ (testing "Top-level payload has the expected keys: website / id / hostname / tag / name / data"
+ (let [{:keys [payload]} (#'metaplow/build-payload :snowplow/dashboard {:event :dashboard-created})]
+ (is (= #{:website :id :hostname :tag :name :data} (set (keys payload))))
+ (is (=? {:website string?
+ :id (analytics.settings/analytics-uuid)
+ :hostname "anonymous.metabase.com"
+ :tag "metabase-instance"}
+ payload))))
+
+ (testing "`data` is enriched with version_tag and plan on every event"
+ (is (=? {:payload {:data {"version_tag" (:tag (version/version))
+ "plan" string?}}}
+ (#'metaplow/build-payload :snowplow/dashboard {:event :dashboard-created}))))
+
+ (testing "Keyword keys and values become snake_case strings"
+ (is (=? {:payload {:name "database.database_connection_successful"
+ :data {"database" "postgres"
+ "database_id" 1
+ "source" "admin"}}}
+ (#'metaplow/build-payload :snowplow/database
+ {:event :database-connection-successful
+ :database :postgres
+ :database-id 1
+ :source :admin}))))
+
+ (testing "hostname matches the FE's anonymized constant regardless of site-url"
+ (mt/with-temporary-setting-values [site-url "https://stats.metabase.com/"]
+ (is (=? {:payload {:hostname "anonymous.metabase.com"}}
+ (#'metaplow/build-payload :snowplow/dashboard {:event :dashboard-created}))))))
+
+(deftest tracking-disabled-test
+ (testing "When metaplow-tracking-enabled is false the event is not enqueued and the call returns false"
+ (mt/with-temporary-setting-values [metaplow-url nil]
+ (let [collector (atom [])]
+ (with-redefs [metaplow/enqueue! (fn [payload]
+ (swap! collector conj payload)
+ true)]
+ (is (false? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created})))
+ (is (empty? @collector)))))))
+
+(deftest pipeline-integration-test
+ (mt/with-temporary-setting-values [metaplow-url "http://fake-metaplow/api/send"
+ anon-tracking-enabled true]
+ (testing "track-event! enqueues onto the real channel and the pipeline worker invokes send-event-with-retries!"
+ (let [received (promise)]
+ (with-redefs [metaplow/send-event-with-retries! (fn [payload]
+ (deliver received payload)
+ :sent)]
+ (is (true? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id 7})))
+ (let [payload (deref received 1000 ::timeout)]
+ (is (not= ::timeout payload)
+ "Worker did not consume the event within 1s")
+ (when (not= ::timeout payload)
+ (is (=? {:type "event"
+ :payload {:name "dashboard.dashboard_created"
+ :data {"dashboard_id" 7}}}
+ payload)))))))
+
+ (testing "Sending 200 events: all of them traverse the pipeline"
+ (let [received (atom [])
+ latch (CountDownLatch. 200)]
+ (with-redefs [metaplow/send-event-with-retries! (fn [payload]
+ (swap! received conj payload)
+ (.countDown latch)
+ :sent)]
+ (doseq [i (range 200)]
+ (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id i}))
+ (is (true? (.await latch 1 TimeUnit/SECONDS))
+ "Pipeline did not process all 200 events within 1s")
+ (is (= 200 (count @received)))
+ (is (= (set (range 200))
+ (set (map #(get-in % [:payload :data "dashboard_id"]) @received)))))))))