The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GRO-1083: Send analytics to Metabase Track by default on self-hosted and OSS instances (#84252)
John Swanson57006c684fc3
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 1cab8dd4fc7..0768922d880 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -180,6 +180,7 @@:module-exports #{analytics.interfaceanalytics.internal-stats}:api #{metabase.analytics.api+ metabase.analytics.api.metaplow-proxymetabase.analytics.api.proxymetabase.analytics.coremetabase.analytics.eventdiff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 632e737b8ab..b84199e8940 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -28,7 +28,7 @@:metabase/discourage-millis-duration 1:metabase/modules 23:metabase/namespace-name 3- :metabase/validate-defendpoint-has-response-schema 426+ :metabase/validate-defendpoint-has-response-schema 427:metabase/validate-defendpoint-query-params-use-kebab-case 48:metabase/validate-defendpoint-route-uses-kebab-case 44:missing-docstring 4diff --git a/.github/actions/e2e-prepare-containers/action.yml b/.github/actions/e2e-prepare-containers/action.ymlindex 0c641701127..927c4d749dd 100644--- a/.github/actions/e2e-prepare-containers/action.yml+++ b/.github/actions/e2e-prepare-containers/action.yml@@ -39,6 +39,10 @@ inputs:description: Snowplow micro containerrequired: falsedefault: false+ metaplow:+ description: Metabase Track recording sink container+ required: false+ default: falseruns:using: "composite"@@ -80,6 +84,7 @@ runs:[ "${{ inputs.mocksaml }}" = true ] && SERVICES+=(saml-idp)[ "${{ inputs.webhook }}" = true ] && SERVICES+=(webhook-tester)[ "${{ inputs.snowplow }}" = true ] && SERVICES+=(snowplow-micro)+ [ "${{ inputs.metaplow }}" = true ] && SERVICES+=(metaplow-sink)for attempt in 1 2 3; doecho "Docker compose up attempt $attempt of 3..."@@ -149,3 +154,9 @@ runs:while ! nc -z localhost 9090; do sleep 1; done &&echo -e "${G}Snowplow micro is up and running!"fi++ if ${{ inputs.metaplow }}; then+ echo -e "${Y}checking Metabase Track sink container..." &&+ while ! nc -z localhost 9091; do sleep 1; done &&+ echo -e "${G}Metabase Track sink is up and running!"+ fidiff --git a/.github/file-paths.yaml b/.github/file-paths.yamlindex bf85f91ba48..995e1e62898 100644--- a/.github/file-paths.yaml+++ b/.github/file-paths.yaml@@ -67,6 +67,7 @@ frontend_specs: &frontend_specs- "frontend/**/tests/**"- "frontend/**/*.unit.*"- "enterprise/frontend/**/*.unit.*"+ - "metaplow/**/*.unit.*"- "jest.config.js"- "jest.base.conf.js"- "jest.esm-packages.js"@@ -126,8 +127,12 @@ ci_scripts:- ".github/file-paths.yaml"- ".github/scripts/**"- ".github/actions/**"- - ".github/workflows/e2e-{test,tests,matrix-builder}.yml"- - ".github/workflows/loki.yml"+ # e2e-containers.unit.spec.ts asserts that every workflow starting one analytics collector+ # container starts the other, and that three files agree on the sink's port. A new workflow is+ # exactly the drift it exists to catch, so this follows all of them, not a list.+ - ".github/workflows/*.{yml,yaml}"+ - "metaplow/**"+ - "deps.edn"- ".storybook/story-files.cjs"- "e2e/runner/read-spec-paths.js"- "e2e/support/cypress.config.js"@@ -270,6 +275,11 @@ snowplow: &snowplow- "bin/lint-snowplow-schemas.bb"- "bin/lint-snowplow-schemas-test.bb"+# The recording sink the e2e suite asserts Metabase Track events against, the counterpart to+# `snowplow/` above.+metaplow: &metaplow+ - "metaplow/**"+# e2e harness/support code (commands, support file, runner). Not in the# instrumented bundle, so it maps to no covered module — a change here forces a# full e2e run (see create-test-plan / e2eInfraTouched).@@ -279,6 +289,7 @@ e2e_infra: &e2e_infra- *frontend_configs- *shared_e2e- *snowplow+ - *metaplow- "e2e/**/!(*.cy.spec.@(js|jsx|ts|tsx))"- "frontend/build/**"- "patches/**"diff --git a/.github/scripts/e2e-containers.unit.spec.ts b/.github/scripts/e2e-containers.unit.spec.tsnew file mode 100644index 00000000000..3e2575c7bb7--- /dev/null+++ b/.github/scripts/e2e-containers.unit.spec.ts@@ -0,0 +1,74 @@+import { readFileSync, readdirSync } from "fs";+import { join } from "path";++import { load } from "js-yaml";++const GITHUB = join(__dirname, "..");+const WORKFLOWS = join(GITHUB, "workflows");+const ROOT = join(GITHUB, "..");++const PREPARE_CONTAINERS = "./.github/actions/e2e-prepare-containers";++type Step = { uses?: string; with?: Record<string, unknown> };+type Workflow = { jobs?: Record<string, { steps?: Step[] }> };++/** Every `e2e-prepare-containers` step across the workflows, labelled with the job it lives in. */+const prepareContainerSteps = () =>+ readdirSync(WORKFLOWS)+ .filter((file) => file.endsWith(".yml") || file.endsWith(".yaml"))+ .flatMap((file) => {+ // `js-yaml` returns `unknown`; the shape we read back is the slice of the workflow schema+ // declared above, and a workflow that does not match it simply contributes no steps.+ const workflow = load(+ readFileSync(join(WORKFLOWS, file), "utf8"),+ ) as Workflow;++ return Object.entries(workflow.jobs ?? {}).flatMap(([job, { steps }]) =>+ (steps ?? [])+ .filter((step) => step.uses === PREPARE_CONTAINERS)+ .map((step) => ({+ where: `${file} / ${job}`,+ with: step.with ?? {},+ })),+ );+ });++const read = (...path: string[]) => readFileSync(join(ROOT, ...path), "utf8");++describe("e2e container setup", () => {+ it("finds the prepare-containers steps it is asserting over", () => {+ expect(prepareContainerSteps().length).toBeGreaterThan(0);+ });++ // The `:e2e` alias in deps.edn points the backend at both analytics collectors unconditionally,+ // so a job that starts one has to start the other. Letting them drift apart leaves a job posting+ // every event at a port nothing is listening on — bounded, thanks to the sender's circuit+ // breaker, but a confusing thing to find in a log.+ it("starts the Metabase Track sink wherever it starts Snowplow micro", () => {+ const missing = prepareContainerSteps()+ .filter(+ (step) => step.with.snowplow === true && step.with.metaplow !== true,+ )+ .map((step) => step.where);++ expect(missing).toEqual([]);+ });++ // Three files have to agree on the sink's port and nothing else ties them together: the alias+ // the backend is started with, the port the container publishes, and the readiness check CI+ // blocks on. The e2e helper hardcodes it a fourth time, but that one is covered by the specs.+ it("agrees on the Metabase Track sink's port", () => {+ const [, port] =+ read("deps.edn").match(+ /"-Dmb\.metaplow\.url=http:\/\/localhost:(\d+)\//,+ ) ?? [];++ expect(port).toBeDefined();+ expect(read("metaplow", "docker-compose.yml")).toContain(+ `"${port}:${port}"`,+ );+ expect(+ read(".github", "actions", "e2e-prepare-containers", "action.yml"),+ ).toContain(`nc -z localhost ${port}`);+ });+});diff --git a/.github/scripts/file-paths.unit.spec.ts b/.github/scripts/file-paths.unit.spec.tsindex 1ce38458c86..07867edc528 100644--- a/.github/scripts/file-paths.unit.spec.ts+++ b/.github/scripts/file-paths.unit.spec.ts@@ -181,6 +181,26 @@ describe("file-paths.yaml", () => {).toBe(false);});+ // The e2e Metabase Track sink lives outside frontend/ and e2e/, so nothing else would pull it+ // into either gate: without these entries a sink-only change runs no tests at all. The e2e gate+ // reaches it through e2e_infra, so a sink-only change also forces a full e2e run.+ it("covers the e2e Metabase Track sink in both the e2e and the frontend unit gates", () => {+ expect(matches("e2e_all", "metaplow/sink.js")).toBe(true);+ expect(matches("e2e_infra", "metaplow/sink.js")).toBe(true);+ expect(matches("frontend_specs", "metaplow/sink.unit.spec.js")).toBe(true);+ });++ // e2e-containers.unit.spec.ts asserts over these, and it runs in the ci-scripts suite.+ it.each([+ ".github/workflows/e2e-test.yml",+ ".github/workflows/embedding-sdk-component-tests.yml",+ ".github/actions/e2e-prepare-containers/action.yml",+ "metaplow/docker-compose.yml",+ "deps.edn",+ ])("runs the CI script tests for %s, which a spec asserts over", (path) => {+ expect(matches("ci_scripts", path)).toBe(true);+ });+it("runs the ratchet check on the ratchets file", () => {expect(matches("project_ratchet_checks", ".clj-kondo/ratchets.edn")).toBe(true,diff --git a/.github/workflows/e2e-coverage-shard.yml b/.github/workflows/e2e-coverage-shard.ymlindex 2858a89ba7d..1edd6d02292 100644--- a/.github/workflows/e2e-coverage-shard.yml+++ b/.github/workflows/e2e-coverage-shard.yml@@ -51,6 +51,7 @@ jobs:openldap: truewebhook: truesnowplow: true+ metaplow: truepostgres: truemysql: truemongo: truediff --git a/.github/workflows/e2e-stress-test-flake-fix.yml b/.github/workflows/e2e-stress-test-flake-fix.ymlindex 0f810905e7b..399e7027a59 100644--- a/.github/workflows/e2e-stress-test-flake-fix.yml+++ b/.github/workflows/e2e-stress-test-flake-fix.yml@@ -167,6 +167,7 @@ jobs:maildev: trueopenldap: truesnowplow: true+ metaplow: truepostgres: ${{ inputs.qa_db }}mysql: ${{ inputs.qa_db }}mongo: ${{ inputs.qa_db }}diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.ymlindex f857247e8b3..439a1003989 100644--- a/.github/workflows/e2e-test.yml+++ b/.github/workflows/e2e-test.yml@@ -107,6 +107,7 @@ jobs:openldap: truewebhook: truesnowplow: true+ metaplow: truepostgres: truemysql: truemongo: truediff --git a/.github/workflows/embedding-sdk-component-tests.yml b/.github/workflows/embedding-sdk-component-tests.ymlindex f31d0538987..2d8183748cd 100644--- a/.github/workflows/embedding-sdk-component-tests.yml+++ b/.github/workflows/embedding-sdk-component-tests.yml@@ -49,6 +49,7 @@ jobs:openldap: falsewebhook: falsesnowplow: true+ metaplow: truepostgres: falsemysql: falsemongo: false@@ -264,6 +265,7 @@ jobs:openldap: falsewebhook: falsesnowplow: true+ metaplow: truepostgres: falsemysql: falsemongo: falsediff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.ymlindex fda9cb0beae..bc1a06b2d7d 100644--- a/.github/workflows/pre-release.yml+++ b/.github/workflows/pre-release.yml@@ -184,6 +184,7 @@ jobs:password: ${{ secrets.DOCKERHUB_TOKEN }}maildev: truesnowplow: true+ metaplow: true- name: Prepare front-end environmentuses: ./.github/actions/prepare-frontend- name: Prepare back-end environmentdiff --git a/deps.edn b/deps.ednindex 5a2afa00ac4..45aa3598912 100644--- a/deps.edn+++ b/deps.edn@@ -415,6 +415,9 @@;; We use Snowplow micro in e2e tests"-Dmb.snowplow.available=true""-Dmb.snowplow.url=http://localhost:9090"+ ;; ...and the recording sink in `metaplow/` for Metabase Track. `metaplow-url`'s production+ ;; default is nil outside production run mode, so e2e has to name the collector itself.+ "-Dmb.metaplow.url=http://localhost:9091/api/send";; See: https://github.com/metabase/metabase/pull/38280"-Dmb.last.analytics.checksum=-1""-Dmb.experimental.search.block.on.queue=true"diff --git a/e2e/support/helpers/e2e-deep-match.js b/e2e/support/helpers/e2e-deep-match.jsnew file mode 100644index 00000000000..402af0447c1--- /dev/null+++ b/e2e/support/helpers/e2e-deep-match.js@@ -0,0 +1,46 @@+export function isDeepMatch(objectOrValue, partialObjectOrValue) {+ if (isMatcher(partialObjectOrValue)) {+ return partialObjectOrValue(objectOrValue);+ }++ const bothAreNotObjects =+ // Check null because typeof null === "object"+ objectOrValue == null ||+ partialObjectOrValue == null ||+ typeof objectOrValue !== "object" ||+ typeof partialObjectOrValue !== "object";++ // Exit condition when calling recursively+ if (bothAreNotObjects) {+ return objectOrValue === partialObjectOrValue;+ }++ for (const [key, value] of Object.entries(partialObjectOrValue)) {+ if (Array.isArray(value)) {+ if (+ !Array.isArray(objectOrValue[key]) ||+ !isArrayDeepMatch(objectOrValue[key], value)+ ) {+ return false;+ }+ } else if (!isDeepMatch(objectOrValue[key], value)) {+ return false;+ }+ }++ return true;+}++function isMatcher(value) {+ return typeof value === "function";+}++function isArrayDeepMatch(array, partialArray) {+ for (const index in partialArray) {+ if (!isDeepMatch(array[index], partialArray[index])) {+ return false;+ }+ }++ return true;+}diff --git a/e2e/support/helpers/e2e-metaplow-helpers.ts b/e2e/support/helpers/e2e-metaplow-helpers.tsnew file mode 100644index 00000000000..cea52854e70--- /dev/null+++ b/e2e/support/helpers/e2e-metaplow-helpers.ts@@ -0,0 +1,59 @@+import { isDeepMatch } from "./e2e-deep-match";++type DeepMatch = Record<string, unknown>;++const METAPLOW_SINK_URL = "http://localhost:9091";+const METAPLOW_INTERVAL = 250;+const METAPLOW_TIMEOUT = 5000;++export interface MetaplowSinkEvent {+ receivedAt: string;+ path: string;+ body: {+ type?: string;+ payload?: Record<string, unknown>;+ } | null;+ raw?: string;+}++export const resetMetaplow = () => {+ sendMetaplowRequest<{ reset: true }>("sink/reset");+};++export const expectMetaplowRequest = (match: DeepMatch, count = 1) => {+ retryForEvents(+ (events) => events.filter((e) => isDeepMatch(e, match)).length === count,+ ).should("be.ok");+};++export const expectMetaplowEvent = (payload: DeepMatch, count = 1) => {+ expectMetaplowRequest({ body: { type: "event", payload } }, count);+};++const sendMetaplowRequest = <T>(url: string) => {+ cy.log("Send a Metabase Track sink request");+ return cy.request<T>({ url: `${METAPLOW_SINK_URL}/${url}`, json: true });+};++type EventsResponse = Cypress.Chainable<Cypress.Response<MetaplowSinkEvent[]>>;++const retryForEvents = (+ condition: (events: MetaplowSinkEvent[]) => boolean,+ timeout = METAPLOW_TIMEOUT,+): EventsResponse => {+ return sendMetaplowRequest<MetaplowSinkEvent[]>("sink/events").then(+ (response) => {+ if (condition(response.body)) {+ return cy.wrap(response);+ }+ if (timeout <= 0) {+ const names = response.body.map((e) => e?.body?.payload?.name);+ throw new Error(+ `Metabase Track sink retry timeout. Events recorded: ${JSON.stringify(names).slice(0, 512)}`,+ );+ }+ cy.wait(METAPLOW_INTERVAL);+ return retryForEvents(condition, timeout - METAPLOW_INTERVAL);+ },+ );+};diff --git a/e2e/support/helpers/e2e-snowplow-helpers.js b/e2e/support/helpers/e2e-snowplow-helpers.jsindex 3ef09279315..51e623cd7a1 100644--- a/e2e/support/helpers/e2e-snowplow-helpers.js+++ b/e2e/support/helpers/e2e-snowplow-helpers.js@@ -1,5 +1,9 @@import { updateSetting } from "e2e/support/helpers";+import { isDeepMatch } from "./e2e-deep-match";++export { isDeepMatch };+const SNOWPLOW_URL = "http://localhost:9090";const SNOWPLOW_INTERVAL = 100;// Snowplow events are POSTed asynchronously and then ingested by snowplow-micro, so a@@ -54,53 +58,6 @@ export const expectUnstructuredSnowplowEvent = (eventData, count = 1) => {);};-export function isDeepMatch(objectOrValue, partialObjectOrValue) {- if (isMatcher(partialObjectOrValue)) {- return partialObjectOrValue(objectOrValue);- }-- const bothAreNotObjects =- // Check null because typeof null === "object"- objectOrValue == null ||- partialObjectOrValue == null ||- typeof objectOrValue !== "object" ||- typeof partialObjectOrValue !== "object";-- // Exit condition when calling recursively- if (bothAreNotObjects) {- return objectOrValue === partialObjectOrValue;- }-- for (const [key, value] of Object.entries(partialObjectOrValue)) {- if (Array.isArray(value)) {- if (- !Array.isArray(objectOrValue[key]) ||- !isArrayDeepMatch(objectOrValue[key], value)- ) {- return false;- }- } else if (!isDeepMatch(objectOrValue[key], value)) {- return false;- }- }-- return true;-}--function isMatcher(value) {- return typeof value === "function";-}--function isArrayDeepMatch(array, partialArray) {- for (const index in partialArray) {- if (!isDeepMatch(array[index], partialArray[index])) {- return false;- }- }-- return true;-}-export const expectNoBadSnowplowEvents = () => {sendSnowplowRequest("micro/bad").its("body").should("deep.equal", []);};diff --git a/e2e/support/helpers/index.ts b/e2e/support/helpers/index.tsindex 66381e21bbe..0426b42c5ce 100644--- a/e2e/support/helpers/index.ts+++ b/e2e/support/helpers/index.ts@@ -39,6 +39,7 @@ export * from "./e2e-explorations-helpers";export * from "./e2e-filter-helpers";export * from "./e2e-ldap-helpers";export * from "./e2e-metabot-helpers";+export * from "./e2e-metaplow-helpers";export * from "./e2e-metric-page-helpers";export * from "./e2e-metrics-viewer-helpers";export * from "./e2e-misc-helpers";diff --git a/e2e/test/scenarios/analytics/metabase-track.cy.spec.ts b/e2e/test/scenarios/analytics/metabase-track.cy.spec.tsnew file mode 100644index 00000000000..74a8f7462d5--- /dev/null+++ b/e2e/test/scenarios/analytics/metabase-track.cy.spec.ts@@ -0,0 +1,39 @@+const { H } = cy;++describe("scenarios > analytics > Metabase Track", () => {+ beforeEach(() => {+ H.restore();+ cy.signInAsAdmin();+ H.enableTracking();+ H.resetMetaplow();+ });++ it("delivers a browser-originated event to the collector", () => {+ cy.visit("/");+ cy.findByTestId("home-page").should("be.visible");++ H.expectMetaplowRequest({+ path: "/api/send",+ body: {+ type: "event",+ payload: {+ name: "pageview",+ url: "http://anonymous.metabase.com/",+ hostname: "anonymous.metabase.com",+ tag: "metabase-instance",+ },+ },+ });+ });++ it("delivers a backend-originated event to the collector", () => {+ H.createDashboard({ name: "Tracked dashboard" }).then(({ body }) => {+ H.expectMetaplowEvent({+ name: "dashboard.dashboard_created",+ hostname: "anonymous.metabase.com",+ tag: "metabase-instance",+ data: { dashboard_id: body.id },+ });+ });+ });+});diff --git a/e2e/test/scenarios/docker-compose.yml b/e2e/test/scenarios/docker-compose.ymlindex 7e8e1fc0c19..9c74f2b1aaf 100644--- a/e2e/test/scenarios/docker-compose.yml+++ b/e2e/test/scenarios/docker-compose.yml@@ -2,6 +2,7 @@ name: metabase-e2einclude:- ../../../snowplow/docker-compose.yml+ - ../../../metaplow/docker-compose.ymlservices:postgres-sample:diff --git a/frontend/lint/config.mjs b/frontend/lint/config.mjsindex 96d3f691d53..bdf8ee21159 100644--- a/frontend/lint/config.mjs+++ b/frontend/lint/config.mjs@@ -547,6 +547,19 @@ const configs = ["no-console": "off",},},+ {+ files: ["metaplow/**/*.js"],+ languageOptions: {+ sourceType: "commonjs",+ globals: {+ ...globals.node,+ },+ },+ rules: {+ "no-console": "off",+ "import/no-commonjs": "off",+ },+ },{files: ["frontend/src/metabase/**/*"],diff --git a/frontend/lint/scripts/run-oxlint.mjs b/frontend/lint/scripts/run-oxlint.mjsindex bf9261bea56..5dea14cc161 100644--- a/frontend/lint/scripts/run-oxlint.mjs+++ b/frontend/lint/scripts/run-oxlint.mjs@@ -9,7 +9,7 @@ const FLAGS = ["0","--report-unused-disable-directives",];-const DEFAULT_PATHS = ["enterprise/frontend", "frontend", "e2e"];+const DEFAULT_PATHS = ["enterprise/frontend", "frontend", "e2e", "metaplow"];export function getOxlintArgs(args, pathExists) {const hasPath = args.some(diff --git a/frontend/lint/tests/oxlint-run.test.mjs b/frontend/lint/tests/oxlint-run.test.mjsindex cc5956408cc..f9b74e7bd69 100644--- a/frontend/lint/tests/oxlint-run.test.mjs+++ b/frontend/lint/tests/oxlint-run.test.mjs@@ -9,7 +9,7 @@ const flags = ["0","--report-unused-disable-directives",];-const defaultPaths = ["enterprise/frontend", "frontend", "e2e"];+const defaultPaths = ["enterprise/frontend", "frontend", "e2e", "metaplow"];const file = "frontend/src/metabase/dev.ts";const pathExists = (arg) => arg === file;diff --git a/frontend/src/embedding-sdk-bundle/analytics/tracker.ts b/frontend/src/embedding-sdk-bundle/analytics/tracker.tsindex 4b023a1fc48..c5d1aba8c54 100644--- a/frontend/src/embedding-sdk-bundle/analytics/tracker.ts+++ b/frontend/src/embedding-sdk-bundle/analytics/tracker.ts@@ -15,8 +15,20 @@ import type { SdkStore } from "embedding-sdk-bundle/store/types";import { getSdkPackageVersion } from "embedding-sdk-shared/lib/get-build-info";import type { MetabaseAuthConfig } from "embedding-sdk-shared/types/auth-config";import { isEmbeddingEajs } from "metabase/embedding-sdk/config";+import { getSettings } from "metabase/settings";import { initMetaplow } from "metabase/utils/metaplow";+const METAPLOW_RELAY_PATH = "/api/metaplow-proxy";++function getMetaplowHostUrl(store: SdkStore, metabaseInstanceUrl: string) {+ const isEnabled = !!getSettings(store.getState())?.[+ "metaplow-tracking-enabled"+ ];+ return isEnabled+ ? `${metabaseInstanceUrl.replace(/\/+$/, "")}${METAPLOW_RELAY_PATH}`+ : null;+}+export function deriveAuthMethod(authConfig: MetabaseAuthConfig,): SdkAuthMethod {@@ -44,6 +56,7 @@ export function useInitSdkTracker(return;}initMetaplow({+ hostUrl: getMetaplowHostUrl(reduxStore, authConfig.metabaseInstanceUrl),// Via frontend/src/embedding-sdk-bundle/analytics/snowplow.ts// Omits userId — unlike the main-app tracker, SDK component usage is tracked at instance granularity;// the analytics-uuid already identifies the account.diff --git a/frontend/src/embedding-sdk-bundle/analytics/tracker.unit.spec.tsx b/frontend/src/embedding-sdk-bundle/analytics/tracker.unit.spec.tsxindex 262300e5867..bbb92cd2ea7 100644--- a/frontend/src/embedding-sdk-bundle/analytics/tracker.unit.spec.tsx+++ b/frontend/src/embedding-sdk-bundle/analytics/tracker.unit.spec.tsx@@ -16,16 +16,24 @@ jest.mock("embedding-sdk-shared/lib/get-build-info", () => ({getSdkPackageVersion: jest.fn(() => "1.2.3"),}));+import { reset } from "@metabase/track";import { renderHook } from "@testing-library/react";+import { createMockState, seedApiQueryCache } from "__support__/state";import { useIsTrackingEnabled } from "embedding-sdk-bundle/analytics/component-events";import {initSdkTracker,trackSdkSimpleEvent,} from "embedding-sdk-bundle/analytics/snowplow";import { setSdkTrackerReady } from "embedding-sdk-bundle/store/reducer";+import type { SdkStore } from "embedding-sdk-bundle/store/types";+import { createMockSdkState } from "embedding-sdk-bundle/test/mocks/state";import type { MetabaseAuthConfig } from "embedding-sdk-shared/types/auth-config";import { EMBEDDING_SDK_IFRAME_EMBEDDING_CONFIG } from "metabase/embedding-sdk/config";+import { trackMetaplowEvent } from "metabase/utils/metaplow";+import Settings from "metabase/utils/settings";+import type { EnterpriseSettings } from "metabase-types/api";+import { createMockSettings } from "metabase-types/api/mocks";import { deriveAuthMethod, useInitSdkTracker } from "./tracker";@@ -33,9 +41,22 @@ const mockUseIsTrackingEnabled = jest.mocked(useIsTrackingEnabled);const mockInitSdkTracker = jest.mocked(initSdkTracker);const mockTrackSdkSimpleEvent = jest.mocked(trackSdkSimpleEvent);-function makeStore() {- // Unjustified type cast. FIXME- return { dispatch: jest.fn(), getState: jest.fn() } as any;+function makeStore(settings: Partial<EnterpriseSettings> = {}) {+ const state = createMockState({ sdk: createMockSdkState() });+ const stateWithSettings = {+ ...state,+ "metabase-api": seedApiQueryCache(state["metabase-api"], [+ {+ endpointName: "getSessionProperties",+ value: createMockSettings(settings),+ },+ ]),+ };+ // The hook only dispatches and reads state; a full SdkStore is not needed.+ return {+ dispatch: jest.fn(),+ getState: () => stateWithSettings,+ } as unknown as SdkStore;}const SSO_AUTH_CONFIG: MetabaseAuthConfig = {@@ -141,3 +162,83 @@ describe("useInitSdkTracker", () => {expect(mockTrackSdkSimpleEvent).not.toHaveBeenCalled();});});++describe("Metabase Track host URL", () => {+ const COLLECTOR_URL = "https://collector.example.com/api/send";+ const RELAY_SEND_URL = "https://metabase.example.com/api/metaplow-proxy/send";+ const trackingOn = { "metaplow-tracking-enabled": true };++ let fetchSpy: jest.SpyInstance;++ beforeEach(() => {+ jest.clearAllMocks();+ mockUseIsTrackingEnabled.mockReturnValue(true);+ mockInitSdkTracker.mockReturnValue(true);+ fetchSpy = jest+ .spyOn(global, "fetch")+ .mockResolvedValue(new Response(null, { status: 200 }));+ });++ afterEach(() => {+ fetchSpy.mockRestore();+ Settings.set("metaplow-url", null);+ reset();+ });++ const getRequestedUrl = () => {+ expect(fetchSpy).toHaveBeenCalledTimes(1);+ return fetchSpy.mock.calls[0][0];+ };++ it("posts through the instance relay instead of the collector", async () => {+ Settings.set("metaplow-url", COLLECTOR_URL);+ renderHook(() =>+ useInitSdkTracker(SSO_AUTH_CONFIG, makeStore(trackingOn), false),+ );++ await trackMetaplowEvent("button_clicked");++ expect(getRequestedUrl()).toBe(RELAY_SEND_URL);+ });++ it("routes through the relay on every instance, Cloud included", async () => {+ renderHook(() =>+ useInitSdkTracker(SSO_AUTH_CONFIG, makeStore(trackingOn), false),+ );++ await trackMetaplowEvent("button_clicked");++ expect(getRequestedUrl()).toBe(RELAY_SEND_URL);+ });++ it("does not double the separator when the instance URL has a trailing slash", async () => {+ renderHook(() =>+ useInitSdkTracker(+ {+ ...SSO_AUTH_CONFIG,+ metabaseInstanceUrl: "https://metabase.example.com/",+ },+ makeStore(trackingOn),+ false,+ ),+ );++ await trackMetaplowEvent("button_clicked");++ expect(getRequestedUrl()).toBe(RELAY_SEND_URL);+ });++ it("does not initialize the tracker when the instance has Metabase Track off", async () => {+ renderHook(() =>+ useInitSdkTracker(+ SSO_AUTH_CONFIG,+ makeStore({ "metaplow-tracking-enabled": false }),+ false,+ ),+ );++ await trackMetaplowEvent("button_clicked");++ expect(fetchSpy).not.toHaveBeenCalled();+ });+});diff --git a/frontend/src/metabase/app.tsx b/frontend/src/metabase/app.tsxindex fc3b2e1225c..68181f8108d 100644--- a/frontend/src/metabase/app.tsx+++ b/frontend/src/metabase/app.tsx@@ -46,7 +46,7 @@ import { captureClickModifierKeys } from "metabase/urls";import { setBasename } from "metabase/utils/basename";import { applyDocumentLocales } from "metabase/utils/boot-locale";import { captureConsoleErrors } from "metabase/utils/errors";-import { initMetaplow } from "metabase/utils/metaplow";+import { getCollectorHostUrl, initMetaplow } from "metabase/utils/metaplow";import { initTracing, rotateTraceId } from "metabase/utils/otel";import { PERFORMANCE_MARKS, markOnce } from "metabase/utils/performance-marks";import MetabaseSettings from "metabase/utils/settings";@@ -114,6 +114,7 @@ async function _init(createSnowplowTracker(() => getUserId(store.getState()));initMetaplow({+ hostUrl: getCollectorHostUrl(),getUserId: () => getUserId(store.getState()),});diff --git a/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.tsx b/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.tsxindex c9207ce7e03..b3735cb3321 100644--- a/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.tsx+++ b/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.tsx@@ -27,6 +27,7 @@ import { PLUGIN_EMBEDDING_IFRAME_SDK } from "metabase/plugins";import { useSelector } from "metabase/redux";import { getSetting } from "metabase/settings";import { Stack } from "metabase/ui";+import { getCollectorHostUrl, initMetaplow } from "metabase/utils/metaplow";import { useParamRerenderKey } from "../hooks/use-param-rerender-key";import { useSdkIframeEmbedEventBus } from "../hooks/use-sdk-iframe-embed-event-bus";@@ -78,6 +79,10 @@ const onSettingsChanged = (settings: SdkIframeEmbedSettings) => {const store = getSdkStore();createSnowplowTracker(() => getUserId(store.getState()));+initMetaplow({+ hostUrl: getCollectorHostUrl(),+ getUserId: () => getUserId(store.getState()),+});export const SdkIframeEmbedRoute = () => {const { embedSettings } = useSdkIframeEmbedEventBus({diff --git a/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.unit.spec.tsx b/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.unit.spec.tsxnew file mode 100644index 00000000000..41ecf177755--- /dev/null+++ b/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.unit.spec.tsx@@ -0,0 +1,22 @@+import { getCollectorHostUrl, initMetaplow } from "metabase/utils/metaplow";++jest.mock("metabase/utils/metaplow", () => ({+ getCollectorHostUrl: jest.fn(() => "https://collector.example.com/api"),+ initMetaplow: jest.fn(),+ trackMetaplowEvent: jest.fn(),+ trackMetaplowPageView: jest.fn(),+}));++describe("SdkIframeEmbedRoute analytics", () => {+ it("initializes Metabase Track for the iframe leg, posting to the collector directly", async () => {+ await import("./SdkIframeEmbedRoute");++ expect(getCollectorHostUrl).toHaveBeenCalled();+ expect(initMetaplow).toHaveBeenCalledWith(+ expect.objectContaining({+ hostUrl: "https://collector.example.com/api",+ getUserId: expect.any(Function),+ }),+ );+ });+});diff --git a/frontend/src/metabase/utils/metaplow.ts b/frontend/src/metabase/utils/metaplow.tsindex 5bb41e4a043..7665c358b11 100644--- a/frontend/src/metabase/utils/metaplow.ts+++ b/frontend/src/metabase/utils/metaplow.ts@@ -17,15 +17,22 @@ const getSanitizedUrl = (url: string) => {};interface MetaplowConfig {+ hostUrl: string | null;getUserId: () => number | undefined;}-export function initMetaplow(config: MetaplowConfig): void {+export function getCollectorHostUrl(): string | null {+ if (!Settings.get("metaplow-tracking-enabled")) {+ return null;+ }const metaplowUrl = Settings.get("metaplow-url");- if (!metaplowUrl) {+ return metaplowUrl ? metaplowUrl.replace(/\/send$/, "") : null;+}++export function initMetaplow({ hostUrl, getUserId }: MetaplowConfig): void {+ if (!hostUrl) {return;}- const hostUrl = metaplowUrl.replace(/\/send$/, "");init({website: METAPLOW_WEBSITE_ID,@@ -59,7 +66,7 @@ export function initMetaplow(config: MetaplowConfig): void {data: {// Unjustified type cast. FIXME...(payload.data as Record<string, unknown> | undefined),- user_id: config.getUserId(),+ user_id: getUserId(),},};},diff --git a/frontend/src/metabase/utils/metaplow.unit.spec.ts b/frontend/src/metabase/utils/metaplow.unit.spec.tsindex 3a9d4b8a03f..46df25fc310 100644--- a/frontend/src/metabase/utils/metaplow.unit.spec.ts+++ b/frontend/src/metabase/utils/metaplow.unit.spec.ts@@ -3,6 +3,7 @@ import { reset } from "@metabase/track";import Settings from "metabase/utils/settings";import {+ getCollectorHostUrl,initMetaplow,trackMetaplowEvent,trackMetaplowPageView,@@ -11,9 +12,13 @@ import {const METAPLOW_URL = "https://metaplow.example.com/api/send";const METAPLOW_WEBSITE_ID = "23eefa30-4c4f-490e-aa4f-084cd23b1561";const ANON_ORIGIN = "http://anonymous.metabase.com";+const RELAY_HOST_URL = "https://app.example.com/api/metaplow-proxy";-const setup = async ({ userId }: { userId?: number } = {}) => {- initMetaplow({ getUserId: () => userId });+const setup = async ({+ userId,+ hostUrl = getCollectorHostUrl(),+}: { userId?: number; hostUrl?: string | null } = {}) => {+ initMetaplow({ hostUrl, getUserId: () => userId });await Promise.resolve();};@@ -21,6 +26,7 @@ describe("metaplow", () => {let fetchSpy: jest.SpyInstance;beforeEach(() => {+ Settings.set("metaplow-tracking-enabled", true);Settings.set("metaplow-url", METAPLOW_URL);fetchSpy = jest.spyOn(global, "fetch")@@ -29,6 +35,7 @@ describe("metaplow", () => {afterEach(() => {fetchSpy.mockRestore();+ Settings.set("metaplow-tracking-enabled", false);Settings.set("metaplow-url", null);reset();});@@ -46,10 +53,26 @@ describe("metaplow", () => {return JSON.parse(init?.body as string);};+ describe("getCollectorHostUrl", () => {+ it("strips the /send suffix the tracker library adds back on", () => {+ expect(getCollectorHostUrl()).toBe("https://metaplow.example.com/api");+ });++ it("is null when no collector is configured", () => {+ Settings.set("metaplow-url", null);+ expect(getCollectorHostUrl()).toBeNull();+ });++ it("is null when the instance has Metabase Track off, even with a collector configured", () => {+ Settings.set("metaplow-tracking-enabled", false);+ expect(getCollectorHostUrl()).toBeNull();+ });+ });+describe("initMetaplow", () => {describe("getUserId", () => {it("injects the user_id into the event data", async () => {- initMetaplow({ getUserId: () => 123 });+ await setup({ userId: 123 });await trackMetaplowEvent("button_clicked");@@ -57,6 +80,23 @@ describe("metaplow", () => {expect(payload.data.user_id).toBe(123);});});++ it("posts to the host URL it is given rather than to the configured collector", async () => {+ await setup({ hostUrl: RELAY_HOST_URL });++ await trackMetaplowEvent("button_clicked");++ expect(fetchSpy).toHaveBeenCalledTimes(1);+ expect(fetchSpy.mock.calls[0][0]).toBe(`${RELAY_HOST_URL}/send`);+ });++ it("does not initialize without a host URL", async () => {+ await setup({ hostUrl: null });++ await trackMetaplowEvent("button_clicked");++ expect(fetchSpy).not.toHaveBeenCalled();+ });});describe("trackMetaplowEvent", () => {diff --git a/metaplow/docker-compose.yml b/metaplow/docker-compose.ymlnew file mode 100644index 00000000000..295ed6c022a--- /dev/null+++ b/metaplow/docker-compose.yml@@ -0,0 +1,11 @@+services:+ metaplow-sink:+ ## Recording stand-in for the Metabase Track collector; see sink.js. Zero dependencies, so a+ ## stock node image running the mounted script is the whole service.+ ## Pinned, like snowplow-micro next door: CI should not pick up a new runtime on its own.+ image: node:22.23.2-alpine+ ports:+ - "9091:9091"+ volumes:+ - .:/sink:ro+ command: node /sink/sink.jsdiff --git a/metaplow/sink.js b/metaplow/sink.jsnew file mode 100644index 00000000000..11079276918--- /dev/null+++ b/metaplow/sink.js@@ -0,0 +1,148 @@+/**+ * Recording sink for Metabase Track (Metaplow) events, used by the end-to-end suite.+ *+ * It stands in for the collector: it accepts the posts the browser tracker and the backend sender+ * make, keeps them in memory, and hands them back so a spec can assert an event actually arrived.+ * Deliberately not a collector — a real one is an application with a database and migrations, which+ * is disproportionate CI surface for asserting that a JSON body arrived. The equivalent for Snowplow+ * is the `snowplow-micro` container in `snowplow/docker-compose.yml`.+ *+ * Routes:+ * POST <anything> record the body and answer `{}` — a JSON object with no `disabled` flag, which+ * is what the tracker library needs to keep sending. Any path is recorded, not+ * just the collector's, so a misconfigured collector URL shows up as a wrong+ * `path` rather than as silence.+ * GET /sink/events everything recorded since the last reset, oldest first+ * GET /sink/reset drop everything recorded+ *+ * No dependencies: the container runs this file straight off a stock `node` image.+ */++const http = require("node:http");++/** Plenty for any one spec, and bounds memory across a whole e2e run that never resets. */+const DEFAULT_MAX_EVENTS = 1000;++/**+ * A real event is a few KB — the tracker truncates strings at 500 characters — and the largest thing+ * the backend sends is the daily instance-statistics payload. 1MB is well clear of both.+ */+const DEFAULT_MAX_BODY_BYTES = 1024 * 1024;++const CORS_HEADERS = {+ // The app posts from a different origin (the instance) than the sink's own.+ "Access-Control-Allow-Origin": "*",+ "Access-Control-Allow-Methods": "GET, POST, OPTIONS",+ // `x-track-cache` is the tracker library's own header; without it the browser blocks the post.+ "Access-Control-Allow-Headers": "content-type, x-track-cache",+ "Access-Control-Max-Age": "86400",+};++function respond(res, status, body) {+ const payload = JSON.stringify(body ?? {});+ res.writeHead(status, {+ ...CORS_HEADERS,+ "Content-Type": "application/json",+ "Content-Length": Buffer.byteLength(payload),+ });+ res.end(payload);+}++/** Reads the whole body, or resolves `null` once it goes over `maxBodyBytes`. */+function readBody(req, maxBodyBytes) {+ return new Promise((resolve, reject) => {+ const chunks = [];+ let size = 0;+ let overflowed = false;++ req.on("data", (chunk) => {+ size += chunk.length;+ if (size > maxBodyBytes) {+ overflowed = true;+ // Keep draining rather than destroying the socket: the client gets a 413 instead of a+ // connection reset, which is far easier to read in a failing spec.+ return;+ }+ chunks.push(chunk);+ });+ req.on("end", () =>+ resolve(overflowed ? null : Buffer.concat(chunks).toString("utf8")),+ );+ req.on("error", reject);+ });+}++function createSink({+ maxEvents = DEFAULT_MAX_EVENTS,+ maxBodyBytes = DEFAULT_MAX_BODY_BYTES,+} = {}) {+ let events = [];++ const record = (path, raw) => {+ let body = null;+ try {+ body = JSON.parse(raw);+ } catch {+ // Left as null, with the text kept in `raw`. A body the collector could not have read is+ // itself worth seeing in a failing spec.+ }+ events.push({+ receivedAt: new Date().toISOString(),+ path,+ body,+ ...(body === null ? { raw } : {}),+ });+ if (events.length > maxEvents) {+ events = events.slice(events.length - maxEvents);+ }+ };++ return http.createServer((req, res) => {+ const path = (req.url ?? "/").split("?")[0];++ if (req.method === "OPTIONS") {+ res.writeHead(204, CORS_HEADERS);+ res.end();+ return;+ }++ if (req.method === "GET" && path === "/sink/events") {+ respond(res, 200, events);+ return;+ }++ if (req.method === "GET" && path === "/sink/reset") {+ events = [];+ respond(res, 200, { reset: true });+ return;+ }++ if (req.method === "POST") {+ readBody(req, maxBodyBytes)+ .then((raw) => {+ if (raw === null) {+ respond(res, 413, { error: "body too large" });+ return;+ }+ record(path, raw);+ // `{}` rather than an empty body: the tracker parses the response and reads `disabled` and+ // `cache` out of it, so it has to be JSON, and it must not say `disabled`.+ respond(res, 200, {});+ })+ .catch(() => respond(res, 400, { error: "could not read body" }));+ return;+ }++ respond(res, 404, { error: "not found" });+ });+}++module.exports = { createSink };++if (require.main === module) {+ const port = Number(process.env.PORT ?? 9091);+ createSink().listen(port, () => {+ // eslint-disable-next-line metabase/no-literal-metabase-strings -- a dev container's startup log, never shown to a user+ console.log(`Metabase Track sink listening on ${port}`);+ });+}diff --git a/metaplow/sink.unit.spec.js b/metaplow/sink.unit.spec.jsnew file mode 100644index 00000000000..55ba2168586--- /dev/null+++ b/metaplow/sink.unit.spec.js@@ -0,0 +1,190 @@+const http = require("node:http");++const { createSink } = require("./sink");++const listen = (sink) =>+ new Promise((resolve) => {+ sink.listen(0, "127.0.0.1", () => resolve(sink.address().port));+ });++const closeSink = (sink) => new Promise((resolve) => sink.close(resolve));++/** Minimal `node:http` client so these tests do not depend on a global `fetch`. */+const request = (port, { method = "GET", path = "/", body, headers = {} }) =>+ new Promise((resolve, reject) => {+ const req = http.request(+ { host: "127.0.0.1", port, method, path, headers },+ (res) => {+ let text = "";+ res.setEncoding("utf8");+ res.on("data", (chunk) => (text += chunk));+ res.on("end", () =>+ resolve({ status: res.statusCode, headers: res.headers, text }),+ );+ },+ );+ req.on("error", reject);+ if (body !== undefined) {+ req.write(body);+ }+ req.end();+ });++const post = (port, payload, path = "/api/send") =>+ request(port, {+ method: "POST",+ path,+ body: typeof payload === "string" ? payload : JSON.stringify(payload),+ headers: { "content-type": "application/json" },+ });++const recorded = async (port) => {+ const { status, text } = await request(port, { path: "/sink/events" });+ expect(status).toBe(200);+ return JSON.parse(text);+};++const eventPayload = (name) => ({ type: "event", payload: { name } });++describe("metaplow sink", () => {+ let sink;+ let port;++ const start = async (options) => {+ sink = createSink(options);+ port = await listen(sink);+ return port;+ };++ afterEach(async () => {+ if (sink) {+ await closeSink(sink);+ sink = undefined;+ }+ });++ it("records a posted event and exposes it", async () => {+ await start();++ await post(port, eventPayload("dashboard.dashboard_created"));++ const events = await recorded(port);+ expect(events).toHaveLength(1);+ expect(events[0]).toMatchObject({+ path: "/api/send",+ body: { type: "event", payload: { name: "dashboard.dashboard_created" } },+ });+ expect(typeof events[0].receivedAt).toBe("string");+ });++ it("answers a post with a JSON body the tracker can parse without disabling itself", async () => {+ await start();++ const { status, headers, text } = await post(+ port,+ eventPayload("pageview"),+ );++ expect(status).toBe(200);+ expect(headers["content-type"]).toMatch(/application\/json/);+ expect(JSON.parse(text).disabled).toBeFalsy();+ });++ it("keeps posts in arrival order", async () => {+ await start();++ await post(port, eventPayload("first"));+ await post(port, eventPayload("second"));++ const events = await recorded(port);+ expect(events.map((e) => e.body.payload.name)).toEqual(["first", "second"]);+ });++ it("clears what it recorded on reset", async () => {+ await start();+ await post(port, eventPayload("pageview"));++ const { status } = await request(port, { path: "/sink/reset" });++ expect(status).toBe(200);+ expect(await recorded(port)).toEqual([]);+ });++ it("does not record its own control requests", async () => {+ await start();++ await request(port, { path: "/sink/reset" });+ await request(port, { path: "/sink/events" });++ expect(await recorded(port)).toEqual([]);+ });++ it("records a post to any path, so a misconfigured collector URL is still visible", async () => {+ await start();++ await post(port, eventPayload("pageview"), "/somewhere/else");++ expect(await recorded(port)).toMatchObject([{ path: "/somewhere/else" }]);+ });++ it("records a body that is not JSON as raw text rather than failing", async () => {+ await start();++ const { status } = await post(port, "not json at all");++ expect(status).toBe(200);+ expect(await recorded(port)).toMatchObject([+ { body: null, raw: "not json at all" },+ ]);+ });++ it("rejects and drops a body over the size cap", async () => {+ await start({ maxBodyBytes: 32 });++ const { status } = await post(port, eventPayload("x".repeat(100)));++ expect(status).toBe(413);+ expect(await recorded(port)).toEqual([]);+ });++ it("retains only the most recent events once the cap is reached", async () => {+ await start({ maxEvents: 2 });++ await post(port, eventPayload("first"));+ await post(port, eventPayload("second"));+ await post(port, eventPayload("third"));++ const events = await recorded(port);+ expect(events.map((e) => e.body.payload.name)).toEqual(["second", "third"]);+ });++ it("allows cross-origin posts from the app, including the tracker's cache header", async () => {+ await start();++ const preflight = await request(port, {+ method: "OPTIONS",+ path: "/api/send",+ headers: {+ origin: "http://localhost:4000",+ "access-control-request-method": "POST",+ "access-control-request-headers": "content-type,x-track-cache",+ },+ });++ expect(preflight.status).toBe(204);+ expect(preflight.headers["access-control-allow-origin"]).toBe("*");+ expect(preflight.headers["access-control-allow-headers"]).toMatch(+ /x-track-cache/i,+ );+ const { headers } = await post(port, eventPayload("pageview"));+ expect(headers["access-control-allow-origin"]).toBe("*");+ });++ it("answers an unknown route with a 404", async () => {+ await start();++ const { status } = await request(port, { path: "/nope" });++ expect(status).toBe(404);+ });+});diff --git a/src/metabase/analytics/api/metaplow_proxy.clj b/src/metabase/analytics/api/metaplow_proxy.cljnew file mode 100644index 00000000000..a7a84d49135--- /dev/null+++ b/src/metabase/analytics/api/metaplow_proxy.clj@@ -0,0 +1,191 @@+(ns metabase.analytics.api.metaplow-proxy+ "Public, anonymous Metabase Track telemetry relay for the Embedding SDK.++ The SDK runs inside the customer's page, where the customer's `connect-src` CSP blocks a direct browser POST to the+ Metabase Track collector. The SDK instead points its tracker at this instance — the same host its data calls+ already go to, so it is already CSP-allowlisted — and we forward the payload to the collector server-side, where no+ browser CSP applies. Same arrangement as [[metabase.analytics.api.proxy]], which does this for Snowplow.++ The route is `/send` rather than `/` because the tracker library composes its endpoint as its configured host URL+ plus `/send`; the SDK configures `<instance>/api/metaplow-proxy` as that host URL.++ Status and body are relayed verbatim: the tracker reads a `disabled` flag and a `cache` token out of the+ collector's response body, so a synthesized response would break that protocol.++ Unlike the Snowplow `tp2` payload, Metabase Track event data is arbitrary nested JSON by design, so there is no+ closed schema to validate it against. We validate the envelope the tracker always wraps it in and bound the rest+ with an explicit size cap.++ Body handling: `wrap-json-body` parses the body before this handler runs, so `body` arrives as a map and we+ re-encode it — not a byte-exact relay. Safe here because the tracker produces the body with `JSON.stringify` and+ the collector reads it as JSON, so a parse→encode round trip is semantically lossless."+ (:require+ [clj-http.client :as http]+ [clj-http.conn-mgr :as conn-mgr]+ [diehard.bulkhead :as dh.bh]+ [diehard.core :as dh]+ [metabase.analytics-interface.core :as analytics]+ [metabase.analytics.settings :as analytics.settings]+ [metabase.api.macros :as api.macros]+ [metabase.request.core :as request]+ [metabase.util :as u]+ [metabase.util.json :as json]+ [metabase.util.log :as log]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]+ [metabase.util.throttle :as u.throttle]+ [throttle.core :as throttle])+ (:import+ (clojure.lang ExceptionInfo)))++(set! *warn-on-reflection* true)++(def ^:private forward-timeout-ms 5000)++;; This endpoint is unauthenticated and makes its outbound call inline on the request thread, so an anonymous caller+;; can hold a request thread per call for as long as the forward takes. Same settings as the throttler+;; on [[metabase.analytics.api.proxy]], the sibling relay, so the two public relays do not differ. This bounds the+;; request *rate* per key, not the number of requests in flight; [[forward-bulkhead]] bounds that.+(def ^:private throttler+ (throttle/make-throttler :metaplow-proxy+ :attempts-threshold 600+ :attempt-ttl-ms (* 60 1000)+ ;; don't punish when throttling, just shed load+ :initial-delay-ms 1000+ :delay-exponent 1))++(def ^:private max-concurrent-forwards+ "Total concurrent forwards allowed in flight at once."+ 8)++(defonce ^:private+ ^{:doc "Process-wide bulkhead bounding concurrent forwards; see [[max-concurrent-forwards]]."}+ forward-bulkhead+ (dh.bh/bulkhead {:concurrency max-concurrent-forwards}))++(defonce ^:private+ ^{:doc "Reuses TCP/TLS connections to the collector across forwards. Sized to [[max-concurrent-forwards]]: the+ bulkhead already caps forwards in flight, and every request goes to one host, so `:default-per-route` is the+ actual cap. Wrapped in `delay` so the pool isn't created until the first forward."}+ connection-manager+ (delay (conn-mgr/make-reusable-conn-manager {:threads max-concurrent-forwards+ :default-per-route max-concurrent-forwards})))++(defn- +throttle+ "Middleware that answers 429 for a source address over the rate limit, before request validation and the forward."+ [handler]+ (fn [request respond raise]+ (try+ ;; The bucket key is only as good as the deployment: `ip-address` reads the source-address header+ ;; (`X-Forwarded-For` by default) unless `MB_NOT_BEHIND_PROXY` is set, so it is the real client behind a proxy+ ;; that overwrites that header, and something a caller can pick for itself on a directly-exposed instance. It+ ;; is still the key every other unauthenticated endpoint here throttles on, and the only one available.+ (throttle/check throttler (request/ip-address request))+ (handler request respond raise)+ (catch ExceptionInfo e+ (if (u.throttle/throttle-exception? e)+ ;; No body, unlike the sibling relay's `{:error ...}`. The tracker assigns `cache` from any response it can+ ;; parse as JSON, so a JSON error body would clear the collector's cache token on every throttled send.+ ;; Every other synthesized status in this ns sends no body for the same reason; `Retry-After` carries the+ ;; detail a caller needs.+ (do+ (analytics/inc! :metabase-analytics-relay/rejected {:relay :metaplow, :reason :throttled})+ (respond (u.throttle/throttle-response e nil)))+ (raise e))))))++(def ^:private max-body-bytes+ "Largest payload we will relay. The tracker truncates strings at 500 characters and objects at 50 properties, so a+ real event is a few KB; 64KB leaves generous headroom while still bounding what an anonymous caller can push+ through the instance."+ (* 64 1024))++(def ^:private cache-token-re+ "Shape a cache token has to have before we pass it upstream. The collector issues the token, but it reaches us by+ way of the customer's page, so treat it as caller-controlled and keep it to a bounded run of token characters."+ #"[A-Za-z0-9._~+/=-]{1,512}")++(def ^:private TrackEnvelope+ "The `{:type :payload}` wrapper the tracker library puts around every send. `identify` is not accepted: nothing in+ Metabase calls it, and an anonymous relay has no reason to carry it. Otherwise deliberately open — `:payload`+ carries browser-derived fields the library may add to over time, and `:data` inside it is arbitrary nested JSON."+ ;; `{:closed false}` is what opts a map out of `defendpoint`'s strip-extra-keys decoding — without it the event+ ;; data would be stripped off before we ever forwarded it.+ [:map {:closed false, ::mr/deliberately-open true, :description "a Metabase Track send envelope"}+ [:type [:enum "event"]]+ [:payload [:map {:closed false, ::mr/deliberately-open true, :description "a Metabase Track event payload"}+ [:website ms/NonBlankString]]]])++(defn- cache-token+ "The tracker echoes the collector's cache token back in `x-track-cache` on later sends, letting the collector skip a+ session lookup. Forward it when it looks like a token, drop it otherwise."+ []+ (when-let [token (some-> (request/current-request) :headers (get "x-track-cache"))]+ (when (re-matches cache-token-re token)+ token)))++(defn- forward!+ [collector-url ^String encoded-body token]+ (try+ (let [response (http/post collector-url+ (cond-> {:body encoded-body+ :content-type :json+ :connection-timeout forward-timeout-ms+ :socket-timeout forward-timeout-ms+ ;; Each redirect hop would get its own full timeout, and the hop target is+ ;; chosen by the collector rather than by us. One request, one destination.+ :redirect-strategy :none+ :throw-exceptions false+ :connection-manager @connection-manager}+ token (assoc :headers {"x-track-cache" token})))]+ {:status (:status response)+ :body (:body response)})+ (catch Exception e+ ;; Debug, not warn: the call rate here is set by anonymous callers, so a collector that is down would otherwise+ ;; flood the log. The error counter is the signal, with a stage label that separates relayed SDK events from+ ;; the instance's own sends in [[metabase.analytics.metaplow]].+ (analytics/inc! :metabase-metaplow/errors {:stage :relay-forward!})+ (log/debugf "metaplow-proxy failed forwarding to %s: %s" collector-url (ex-message e))+ ;; The tracker ignores the response on failure, but a fake 2xx would still be a lie about delivery.+ {:status 502 :body nil})))++(defn- do-with-forward-bulkhead+ "Run `thunk` holding a [[forward-bulkhead]] permit, answering 503 when the relay already has+ [[max-concurrent-forwards]] forwards in flight."+ [thunk]+ (try+ (dh/with-bulkhead {:bulkhead forward-bulkhead, :max-wait-ms 0}+ (thunk))+ (catch ExceptionInfo e+ (if (:bulkhead (ex-data e))+ (do+ (analytics/inc! :metabase-analytics-relay/rejected {:relay :metaplow, :reason :shed})+ (log/debug "metaplow-proxy shed a forward: too many already in flight")+ {:status 503 :body nil})+ (throw e)))))++;; No response schema: the body is an opaque relay of the collector's own response. The tracker reads its `disabled`+;; flag and `cache` token straight out of it, so a fabricated schema would be misleading.+;;+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}+(api.macros/defendpoint :post "/send"+ "Forward a Metabase Track payload to the configured collector server-side, bypassing the customer page's+ `connect-src` CSP."+ [_route-params+ _query-params+ body :- TrackEnvelope]+ (if-not (analytics.settings/metaplow-tracking-enabled)+ ;; Gated on the same switch as the instance's own emission — anonymous tracking on and a collector configured.+ ;; An admin who opted out must not have the instance relaying a caller's events either.+ {:status 503 :body nil}+ (let [encoded (json/encode body)]+ (if (> (alength (u/string-to-bytes encoded)) max-body-bytes)+ {:status 413 :body nil}+ ;; `cache-token` reads the current request, so it has to be called on this thread, not inside the forward.+ (let [token (cache-token)]+ (do-with-forward-bulkhead+ (fn []+ (forward! (analytics.settings/metaplow-url) encoded token))))))))++(def ^{:arglists '([request respond raise])} routes+ "Routes for the public Metabase Track telemetry relay."+ (api.macros/ns-handler *ns* +throttle))diff --git a/src/metabase/analytics/api/proxy.clj b/src/metabase/analytics/api/proxy.cljindex 8133015063f..ac02874eb02 100644--- a/src/metabase/analytics/api/proxy.clj+++ b/src/metabase/analytics/api/proxy.clj@@ -10,14 +10,68 @@(base64 blobs survive a parse→encode round-trip), and v1 pins `eventMethod: \"post\"` with a JSON content-type."(:require[clj-http.client :as http]+ [clj-http.conn-mgr :as conn-mgr]+ [diehard.bulkhead :as dh.bh]+ [diehard.core :as dh]+ [metabase.analytics-interface.core :as analytics][metabase.analytics.settings :as analytics.settings][metabase.api.macros :as api.macros]+ [metabase.request.core :as request][metabase.util.json :as json][metabase.util.log :as log]- [metabase.util.malli.schema :as ms]))+ [metabase.util.malli.schema :as ms]+ [metabase.util.throttle :as u.throttle]+ [throttle.core :as throttle])+ (:import+ (clojure.lang ExceptionInfo)))(def ^:private forward-timeout-ms 5000)+;; This endpoint is unauthenticated and makes its outbound call inline on the request thread, so an anonymous caller+;; can hold a request thread per call for as long as the forward takes.+(def ^:private throttler+ (throttle/make-throttler :analytics-proxy+ :attempts-threshold 600+ :attempt-ttl-ms (* 60 1000)+ ;; don't punish when throttling, just shed load+ :initial-delay-ms 1000+ :delay-exponent 1))++(def ^:private max-concurrent-forwards+ "Forwards allowed in flight at once."+ 8)++(defonce ^:private+ ^{:doc "Process-wide bulkhead bounding concurrent forwards; see [[max-concurrent-forwards]]."}+ forward-bulkhead+ (dh.bh/bulkhead {:concurrency max-concurrent-forwards}))++(defonce ^:private+ ^{:doc "Reuses TCP/TLS connections to the collector across forwards. Sized to [[max-concurrent-forwards]]: the+ bulkhead already caps forwards in flight, and every request goes to one host, so `:default-per-route` is the+ actual cap. Wrapped in `delay` so the pool isn't created until the first forward."}+ connection-manager+ (delay (conn-mgr/make-reusable-conn-manager {:threads max-concurrent-forwards+ :default-per-route max-concurrent-forwards})))++(defn- +throttle+ "Middleware that answers 429 for a source address over the rate limit, before request validation and the forward."+ [handler]+ (fn [request respond raise]+ (try+ ;; The bucket key is only as good as the deployment: `ip-address` reads the source-address header+ ;; (`X-Forwarded-For` by default) unless `MB_NOT_BEHIND_PROXY` is set, so it is the real client behind a proxy+ ;; that overwrites that header, and something a caller can pick for itself on a directly-exposed instance. It+ ;; is still the key every other unauthenticated endpoint here throttles on, and the only one available.+ (throttle/check throttler (request/ip-address request))+ (handler request respond raise)+ (catch ExceptionInfo e+ (if (u.throttle/throttle-exception? e)+ (do+ (analytics/inc! :metabase-analytics-relay/rejected {:relay :snowplow, :reason :throttled})+ (respond (u.throttle/throttle-response e {:error (ex-message e)})))+ (raise e))))))+(def ^:private Tp2Payload"Schema for the Snowplow `tp2` request envelope, mirroring the collector's own `payload_data` JSON schema: the igluURI of that schema plus one entry per event, keyed by tracker field name (`e`, `p`, `tv`, `ue_px`, ...). The tracker@@ -26,6 +80,37 @@[:schema ms/NonBlankString][:data [:sequential {:max 100} (ms/string-keyed-map :string)]]])+(defn- forward!+ [collector-url body]+ (try+ (let [response (http/post collector-url+ {:body (json/encode body)+ :content-type :json+ :connection-timeout forward-timeout-ms+ :socket-timeout forward-timeout-ms+ :throw-exceptions false+ :connection-manager @connection-manager})]+ {:status (:status response)+ :body (:body response)})+ (catch Exception e+ (log/errorf "analytics-proxy failed forwarding to %s: %s" collector-url (ex-message e))+ {:status 502 :body nil})))++(defn- do-with-forward-bulkhead+ "Run `thunk` holding a [[forward-bulkhead]] permit, answering 503 when the relay already has+ [[max-concurrent-forwards]] forwards in flight."+ [thunk]+ (try+ (dh/with-bulkhead {:bulkhead forward-bulkhead, :max-wait-ms 0}+ (thunk))+ (catch ExceptionInfo e+ (if (:bulkhead (ex-data e))+ (do+ (analytics/inc! :metabase-analytics-relay/rejected {:relay :snowplow, :reason :shed})+ (log/debug "analytics-proxy shed a forward: too many already in flight")+ {:status 503 :body nil})+ (throw e)))))+;; No response schema: the body is an opaque relay of the collector's own response. The tracker drives its retry;; outbox off the HTTP status, so we forward status + body verbatim; a fabricated schema would be misleading.;;@@ -37,17 +122,10 @@_query-paramsbody :- Tp2Payload](let [collector-url (str (analytics.settings/snowplow-url) "/com.snowplowanalytics.snowplow/tp2")]- (try- (let [response (http/post collector-url- {:body (json/encode body)- :content-type :json- :connection-timeout forward-timeout-ms- :socket-timeout forward-timeout-ms- :throw-exceptions false})]- ;; Relay status + body unchanged: 2xx clears the tracker's event, non-2xx schedules a retry.- {:status (:status response)- :body (:body response)})- (catch Exception e- ;; 502 is retryable by the tracker; a fake 2xx would silently drop the event.- (log/errorf "analytics-proxy failed forwarding to %s: %s" collector-url (ex-message e))- {:status 502 :body nil}))))+ (do-with-forward-bulkhead+ (fn []+ (forward! collector-url body)))))++(def ^{:arglists '([request respond raise])} routes+ "Routes for the public Snowplow telemetry relay."+ (api.macros/ns-handler *ns* +throttle))diff --git a/src/metabase/analytics/core.clj b/src/metabase/analytics/core.cljindex 96abe2d30da..cbeda9e7893 100644--- a/src/metabase/analytics/core.clj+++ b/src/metabase/analytics/core.clj@@ -53,7 +53,8 @@[metabase.analytics.settingsanon-tracking-enabledanon-tracking-enabled!- instance-creation]+ instance-creation+ metaplow-url][metabase.analytics.eventtrack-event!][metabase.analytics.statsdiff --git a/src/metabase/analytics/metaplow.clj b/src/metabase/analytics/metaplow.cljindex a1b02d07c58..68fd00e0387 100644--- a/src/metabase/analytics/metaplow.clj+++ b/src/metabase/analytics/metaplow.clj@@ -2,9 +2,12 @@"Sends analytics events to a Metaplow (Umami-compatible) collector.`track-event!` builds a payload, offers it onto an async send queue, and returns. A `pipeline-blocking` worker pool- drains the queue and POSTs each event to the collector's `/api/send` endpoint, retrying transient failures via- [[metabase.util.retry/with-retry]]. The HTTP connection pool is sized to match the worker count so workers never- contend for slots.+ drains the queue and POSTs each event to the collector's `/api/send` endpoint. The HTTP connection pool is sized to+ match the worker count so workers never contend for slots.++ Delivery is bounded per event by [[retry-policy]] and across events by [[breaker]]. Failed sends log at debug only;+ the `:metabase-metaplow/errors` counter, labelled by `:stage`, is the operational signal. The breaker's state+ changes log at warn and info, at most about once a minute.Emission is gated by [[metabase.analytics.settings/metaplow-tracking-enabled]]. The public entry point used by therest of the codebase is [[metabase.analytics.event/track-event!]], which fans out to both Snowplow and Metaplow."@@ -13,6 +16,8 @@[clj-http.conn-mgr :as conn-mgr][clojure.core.async :as a][clojure.walk :as walk]+ [diehard.circuit-breaker :as dh.cb]+ [diehard.core :as dh][metabase.analytics-interface.core :as analytics][metabase.analytics.settings :as analytics.settings][metabase.analytics.snowplow :as snowplow]@@ -23,7 +28,9 @@[metabase.util.malli :as mu][metabase.util.malli.schema :as ms][metabase.util.retry :as retry]- [metabase.version.core :as version]))+ [metabase.version.core :as version])+ (:import+ (dev.failsafe CircuitBreakerOpenException)))(set! *warn-on-reflection* true)@@ -103,30 +110,84 @@:connection-manager @connection-manager})(catch Throwable e(analytics/inc! :metabase-metaplow/errors {:stage :send-event!})- (log/warnf "Connection failure sending Metaplow event: %s" (ex-message e))+ (log/debugf "Connection failure sending Metaplow event: %s" (ex-message e)){:status -1})))+(defn- delivered?+ "Truthy when the collector accepted the event, i.e. the response is a 2xx."+ [{:keys [status]}]+ (and (some? status) (<= 200 status 299)))+(defn- retryable-response?- "Truthy when a `send-event!` response should trigger another attempt: non-2xx and not one of the no-retry+ "Truthy when a `send-event!` response should trigger another attempt: not accepted, and not one of the no-retrystatus codes that Snowplow also gives up on (400/401/403/410/422)."- [{:keys [status]}]+ [{:keys [status] :as response}](and (some? status)- (not (<= 200 status 299))+ (not (delivered? response))(not (contains? no-retry-status-codes status))))+(defn- failed-attempt?+ "Truthy when one attempt at [[send-event!]] counts as a failure, for both retry and circuit-breaking purposes."+ [response ex]+ ;; a no-retry status is not a failure here: the collector answered, so it is reachable, it just won't take this event+ (or (some? ex) (retryable-response? response)))++(def ^:private retry-policy+ "Bounds the delivery of one event: at most three attempts, 500ms to 2s apart, with no further attempt started more+ than 5s in.++ That bounds the retrying, not the wall clock. `:max-duration-ms` is only tested between attempts, so an event whose+ first attempt stalls out on [[send-event!]]'s connect and socket timeouts gets one attempt rather than three, and+ can hold a worker for around twice those timeouts."+ {:max-retries 2+ :backoff-ms [500 2000]+ :max-duration-ms 5000+ :retry-if failed-attempt?})++(def ^:private breaker-config+ "Opens after ten consecutive failed events, rejects every event for the next 60 seconds, then admits one probe and+ closes again if it succeeds. Counted in failed events, not failed attempts — see [[breaker]]."+ {:failure-threshold 10+ :delay-ms (u/seconds->ms 60)+ :success-threshold 1+ :fail-if failed-attempt?+ :on-open (fn [_] (log/warn "Metaplow collector unreachable; opening circuit breaker"))+ :on-half-open (fn [_] (log/info "Metaplow circuit breaker half-open; probing the collector"))+ :on-close (fn [_] (log/info "Metaplow collector reachable again; closing circuit breaker"))})++(defn- new-breaker+ "A circuit breaker built from [[breaker-config]], with `overrides` merged over it."+ [overrides]+ (dh.cb/circuit-breaker (merge breaker-config overrides)))++(defonce ^:private+ ^{:doc "Process-wide circuit breaker guarding the collector, shared by every worker, so that once it is open they+ all reject immediately."}+ breaker+ (new-breaker nil))+(defn- send-event-with-retries!- "Send an event, retrying with exponential backoff on connection failures or retryable HTTP statuses (via- [[metabase.util.retry/with-retry]]). Returns `:sent` on success and `:error` on failure (never nil), so callers can- reliably distinguish outcomes."+ "Send one event under [[retry-policy]], guarded by [[breaker]]. Returns `:sent` when the collector accepted the+ event, `:error` when it did not, and `:rejected` when the breaker was open and nothing was sent — never nil, so+ callers can tell the three apart."[payload](try- (retry/with-retry {:retry-if (fn [response _ex] (retryable-response? response))}- (send-event! payload))- :sent- (catch Throwable e- (analytics/inc! :metabase-metaplow/errors {:stage :send-event-with-retries!})- (log/warnf "Error sending Metaplow event: %s" (ex-message e))- :error)))+ ;; the breaker wraps the retry rather than being passed to it as Diehard's `:circuit-breaker` option, which would+ ;; compose it inside the retry and count one failure per failed attempt instead of one per failed event+ (let [response (dh/with-circuit-breaker breaker+ (retry/with-retry retry-policy+ (send-event! payload)))]+ (if (delivered? response)+ :sent+ (do (analytics/inc! :metabase-metaplow/errors {:stage :send-event-with-retries!})+ (log/debugf "Giving up on Metaplow event; last status %s" (:status response))+ :error)))+ ;; `send-event!` swallows everything and the retry returns the last response rather than throwing when it gives+ ;; up on one, so the breaker is the only thing in the body that throws+ (catch CircuitBreakerOpenException _e+ (analytics/inc! :metabase-metaplow/errors {:stage :circuit-breaker-open})+ (log/debug "Metaplow circuit breaker is open; dropping event without attempting a send")+ :rejected)))(defonce ^:private^{:doc "Individual payloads queued by `track-event!` land here. Bounded chan of size `queue-size`; new events arediff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 26beaff37b3..d2f3c62291e 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -927,6 +927,10 @@(prometheus/counter :metabase-metaplow/errors{:description "Metaplow event pipeline errors by stage.":labels [:stage]})+ ;; Not labelled by source address: the relays are anonymous and public, so the caller would pick the label value.+ (prometheus/counter :metabase-analytics-relay/rejected+ {:description "Public telemetry relay requests rejected without forwarding, by relay and reason."+ :labels [:relay :reason]})(prometheus/gauge :metabase-memoize/cache-size{:description "Number of entries currently held in a monitored in-memory memoization cache.":labels [:cache]})])diff --git a/src/metabase/analytics/settings.clj b/src/metabase/analytics/settings.cljindex 3ef4b9f87dc..7a46d0d9ce0 100644--- a/src/metabase/analytics/settings.clj+++ b/src/metabase/analytics/settings.clj@@ -3,6 +3,7 @@[java-time.api :as t][metabase.analytics.db :as analytics.db][metabase.config.core :as config]+ [metabase.premium-features.core :as premium-features][metabase.settings.core :as setting :refer [defsetting]][metabase.util.date-2 :as u.date][metabase.util.experiment :as experiment]@@ -45,23 +46,35 @@:visibility :public:doc false)-(defsetting metaplow-tracking-enabled- (deferred-tru- (str "Boolean indicating whether analytics events are being sent to Metaplow. "- "True if anonymous tracking is enabled for this instance, and a Metaplow collector URL is set."))- :type :boolean- :getter (fn [] (boolean (and (anon-tracking-enabled)- (setting/get-value-of-type :string :metaplow-url))))- :visibility :public- :doc false)+(defn- default-metaplow-url+ "The Metabase Track collector in production run mode, and nil in every other run mode — dev, test and e2e runs must+ not send real telemetry. Air-gapped instances get no default either; an explicitly set URL still applies."+ []+ (when (and config/is-prod?+ (not (premium-features/airgap-enabled)))+ "https://product-analytics-ingestion.metabase.com/api/send"))(defsetting metaplow-url(deferred-tru "The URL of the Metaplow collector to send analytics events to."):encryption :when-encryption-key-set+ ;; resolved per read rather than as `:default`, which is fixed at load time, before the token is known+ :getter (fn []+ (or (setting/get-value-of-type :string :metaplow-url)+ (default-metaplow-url))):visibility :public:audit :never:doc false)+(defsetting metaplow-tracking-enabled+ (deferred-tru+ (str "Boolean indicating whether analytics events are being sent to Metaplow. "+ "True if anonymous tracking is enabled for this instance, and a Metaplow collector URL is set or defaulted."))+ :type :boolean+ :getter (fn [] (boolean (and (anon-tracking-enabled)+ (metaplow-url))))+ :visibility :public+ :doc false)+(defsetting snowplow-url(deferred-tru "The URL of the Snowplow collector to send analytics events to."):encryption :when-encryption-key-setdiff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.cljindex 45aa989e514..d6e7c724600 100644--- a/src/metabase/analytics/snowplow.clj+++ b/src/metabase/analytics/snowplow.clj@@ -247,19 +247,35 @@[](TrackerConfiguration. "sp" "metabase"))+(def ^:private http-timeout-ms+ "Connect, socket, and connection-request timeout for the emitter's HTTP client, in milliseconds."+ 5000)++(defn- request-config+ ^RequestConfig []+ (-> (RequestConfig/custom)+ ;; Set cookie spec to `STANDARD` to avoid warnings about an invalid cookie+ ;; header in request response (PR #24579)+ (.setCookieSpec CookieSpecs/STANDARD)+ ;; All three default to no timeout, so without these a collector that accepts packets but never+ ;; responds wedges an emitter thread for the life of the process.+ (.setConnectTimeout http-timeout-ms)+ (.setSocketTimeout http-timeout-ms)+ (.setConnectionRequestTimeout http-timeout-ms)+ (.build)))++(defn- http-client-adapter+ "The HTTP transport the Snowplow emitter's threads use, pointed at `url`."+ ^ApacheHttpClientAdapter [url]+ (let [client (-> (HttpClients/custom)+ (.setConnectionManager (PoolingHttpClientConnectionManager.))+ (.setDefaultRequestConfig (request-config))+ (.build))]+ (ApacheHttpClientAdapter. url client)))+(defn- network-config[]- (let [request-config (-> (RequestConfig/custom)- ;; Set cookie spec to `STANDARD` to avoid warnings about an invalid cookie- ;; header in request response (PR #24579)- (.setCookieSpec CookieSpecs/STANDARD)- (.build))- client (-> (HttpClients/custom)- (.setConnectionManager (PoolingHttpClientConnectionManager.))- (.setDefaultRequestConfig request-config)- (.build))- http-client-adapter (ApacheHttpClientAdapter. (analytics.settings/snowplow-url) client)]- (NetworkConfiguration. http-client-adapter)))+ (NetworkConfiguration. (http-client-adapter (analytics.settings/snowplow-url))))(defn- emitter-config[]diff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.cljindex 25d696a383a..9cc03a4f82f 100644--- a/src/metabase/api_routes/routes.clj+++ b/src/metabase/api_routes/routes.clj@@ -7,6 +7,7 @@[metabase.agent-api.query-guards :as agent-api.query-guards][metabase.ai-tracing.api][metabase.analytics.api]+ [metabase.analytics.api.metaplow-proxy][metabase.analytics.api.proxy][metabase.api-keys.api][metabase.api.docs]@@ -86,6 +87,7 @@metabase.agent-api.api/keep-memetabase.ai-tracing.api/keep-memetabase.analytics.api/keep-me+ metabase.analytics.api.metaplow-proxy/keep-memetabase.analytics.api.proxy/keep-memetabase.api-keys.api/keep-memetabase.api.util/keep-me@@ -194,7 +196,7 @@"/agent" (metabase.agent-api.api/+agent-api-enabled metabase.agent-api.api/routes)"/alert" (+auth metabase.pulse.api/alert-routes)"/analytics" (+auth 'metabase.analytics.api)- "/analytics-proxy" (+public-exceptions 'metabase.analytics.api.proxy)+ "/analytics-proxy" (+public-exceptions metabase.analytics.api.proxy/routes)"/api-key" (+auth 'metabase.api-keys.api)"/automagic-dashboards" (+auth metabase.xrays.api/automagic-dashboards-routes)"/bookmark" (+auth 'metabase.bookmarks.api)@@ -248,6 +250,7 @@"/measure" (+auth 'metabase.measures.api)"/metabase-mcp" (metabase.mcp.v2.api/+mcp-enabled metabase.mcp.v2.api/handler)"/metabot" metabase.metabot.api/routes+ "/metaplow-proxy" (+public-exceptions metabase.analytics.api.metaplow-proxy/routes)"/metric" (+auth 'metabase.metrics.api)"/model-index" (+auth 'metabase.indexed-entities.api)"/native-query-snippet" (+auth 'metabase.native-query-snippets.api)diff --git a/src/metabase/server/middleware/security.clj b/src/metabase/server/middleware/security.cljindex 6dba27e9990..90030cf9b52 100644--- a/src/metabase/server/middleware/security.clj+++ b/src/metabase/server/middleware/security.clj@@ -342,7 +342,7 @@(when (analytics/anon-tracking-enabled)(setting/get-value-of-type :string :snowplow-url))(when (analytics/anon-tracking-enabled)- (setting/get-value-of-type :string :metaplow-url))+ (analytics/metaplow-url));; Webpack dev server(when config/is-dev?(str "*:" frontend-dev-port " ws://*:" frontend-dev-port))diff --git a/test/metabase/analytics/api/metaplow_proxy_test.clj b/test/metabase/analytics/api/metaplow_proxy_test.cljnew file mode 100644index 00000000000..cf3ccdbd049--- /dev/null+++ b/test/metabase/analytics/api/metaplow_proxy_test.clj@@ -0,0 +1,233 @@+(ns metabase.analytics.api.metaplow-proxy-test+ "Integration tests for the public Metabase Track telemetry relay (`POST /api/metaplow-proxy/send`).++ Mirrors `metabase.analytics.api.proxy-test`, which does the same for the Snowplow relay: `clj-http.fake` stands in+ for the collector and `metabase.test.http-client/client` issues anonymous requests."+ (:require+ [clj-http.fake :as fake]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [diehard.bulkhead :as dh.bh]+ [metabase.analytics.api.metaplow-proxy :as analytics.api.metaplow-proxy]+ [metabase.test :as mt]+ [metabase.test.http-client :as client]+ [metabase.util.json :as json]+ [throttle.core :as throttle]))++(set! *warn-on-reflection* true)++(def ^:private collector-url "http://fake-collector.test/api/send")++(def ^:private sample-payload+ "A minimal Metabase Track envelope: the `{:type :payload}` wrapper the tracker library always sends."+ {:type "event"+ :payload {:website "23eefa30-4c4f-490e-aa4f-084cd23b1561"+ :name "button_clicked"+ :url "http://anonymous.metabase.com/dashboard/1"+ :data {:nested {:arbitrary ["json" 1 true]}}}})++(defn- with-collector!+ "Pin `metaplow-url` to the fake collector, serve `response` from it, and call `f` with an atom holding the inbound+ request — still nil if nothing was forwarded."+ [response f]+ (let [captured (atom nil)]+ (mt/with-temporary-setting-values [metaplow-url collector-url]+ (fake/with-fake-routes+ {collector-url (fn [request]+ (reset! captured request)+ response)}+ (f captured)))))++(deftest anonymous-public-route-test+ (testing "an anonymous POST reaches the handler (no auth required) and gets the collector's 2xx back"+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [_captured]+ ;; client/client with no user = anonymous. A 401 here would mean the route isn't public.+ (is (= {} (client/client :post 200 "metaplow-proxy/send" sample-payload)))))))++(deftest forwards-payload-to-collector-test+ (testing "the handler forwards the body to the configured collector as JSON, semantically unchanged"+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [captured]+ (client/client :post 200 "metaplow-proxy/send" sample-payload)+ (testing "outbound URL is the configured collector URL"+ (is (= collector-url+ (str (name (:scheme @captured)) "://" (:server-name @captured) (:uri @captured)))))+ (testing "outbound body round-trips to the same JSON the SDK sent (re-encode is semantically lossless)"+ (is (= (json/decode (json/encode sample-payload))+ (json/decode (slurp (:body @captured))))))+ (testing "outbound content-type is JSON"+ (is (re-find #"(?i)application/json"+ (get-in @captured [:headers "content-type"] ""))))))))++(deftest relays-status-and-body-verbatim-test+ (testing "the tracker's disable/cache protocol needs the collector's own response, not a synthesized one"+ (testing "a 2xx body is relayed unchanged"+ (with-collector! {:status 200 :headers {} :body "{\"cache\":\"abc123\",\"disabled\":false}"}+ (fn [_captured]+ (is (= {:cache "abc123" :disabled false}+ (client/client :post 200 "metaplow-proxy/send" sample-payload))))))+ (testing "a non-2xx status and body are relayed unchanged"+ (with-collector! {:status 400 :headers {} :body "{\"error\":\"bad payload\"}"}+ (fn [_captured]+ (is (= {:error "bad payload"}+ (client/client :post 400 "metaplow-proxy/send" sample-payload))))))))++(deftest forwards-cache-token-test+ (testing "the tracker echoes the collector's cache token in `x-track-cache`; it is forwarded so the collector can+ skip a session lookup"+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [captured]+ (client/client :post 200 "metaplow-proxy/send"+ {:request-options {:headers {"x-track-cache" "abc123"}}}+ sample-payload)+ (is (= "abc123" (get-in @captured [:headers "x-track-cache"]))))))+ (testing "a cache token that is not a plain token string is dropped rather than forwarded"+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [captured]+ (client/client :post 200 "metaplow-proxy/send"+ {:request-options {:headers {"x-track-cache" "not a token"}}}+ sample-payload)+ (is (nil? (get-in @captured [:headers "x-track-cache"])))))))++(deftest rejects-invalid-envelope-test+ (testing "the envelope is validated even though the event data inside it is arbitrary JSON"+ (doseq [[what payload] {"a missing payload" {:type "event"}+ "a missing type" {:payload {:website "abc"}}+ "an unknown type" {:type "pageview" :payload {:website "abc"}}+ "an identify call" {:type "identify" :payload {:website "abc"}}+ "a blank website" {:type "event" :payload {:website ""}}+ "a payload that is not a map" {:type "event" :payload "nope"}}]+ (testing what+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [captured]+ (client/client :post 400 "metaplow-proxy/send" payload)+ (is (nil? @captured) "nothing should reach the collector")))))))++(deftest body-size-cap-test+ (testing "an oversized body is rejected with 413 and never reaches the collector"+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [captured]+ ;; well past the 64KB cap+ (client/client :post 413 "metaplow-proxy/send"+ (assoc-in sample-payload [:payload :data :blob] (str/join (repeat 100000 "x"))))+ (is (nil? @captured) "nothing should reach the collector"))))+ (testing "a payload comfortably under the cap is relayed"+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [_captured]+ (is (= {} (client/client :post 200 "metaplow-proxy/send"+ (assoc-in sample-payload [:payload :data :blob] (str/join (repeat 1000 "x"))))))))))++(deftest gated-on-metaplow-tracking-test+ (testing "with no collector URL configured there is nothing to relay to"+ (mt/with-temporary-setting-values [metaplow-url nil]+ (client/client :post 503 "metaplow-proxy/send" sample-payload)))+ (testing "an instance that opted out of anonymous tracking does not relay a caller's events either"+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [captured]+ (mt/with-temporary-setting-values [anon-tracking-enabled false]+ (client/client :post 503 "metaplow-proxy/send" sample-payload))+ (is (nil? @captured) "nothing should reach the collector")))))++(deftest unreachable-collector-returns-502-test+ (testing "when the collector is unreachable the relay returns 502 rather than throwing"+ (mt/with-temporary-setting-values [metaplow-url collector-url]+ (fake/with-fake-routes+ {collector-url (fn [_request]+ (throw (java.net.ConnectException. "connection refused")))}+ (client/client :post 502 "metaplow-proxy/send" sample-payload)))))++(deftest no-client-ip-forwarded-test+ (testing "the relay does not add X-Forwarded-For (the payload is already anonymized by the tracker)"+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [captured]+ (client/client :post 200 "metaplow-proxy/send" sample-payload)+ (is (nil? (get-in @captured [:headers "x-forwarded-for"])))))))++(def ^:private throttled-response+ "A throttled response carries `Retry-After` and no body: the tracker assigns its cache token from any response body+ it can parse, so a body here would clear the collector's token on every throttled send."+ {:status 429+ :headers {"Retry-After" string?}+ :body nil})++(defn- request-options+ "Request options that make `ip` the source address the server derives for the request. `ip-address` reads+ `X-Forwarded-For` by default, which is also why the throttle bucket is only as trustworthy as the deployment."+ [ip]+ {:request-options {:headers {"x-forwarded-for" ip}}})++(defmacro ^:private with-one-request-budget+ "Pin the relay throttler to a single request per source address, so one request exhausts a caller's budget."+ [& body]+ `(with-redefs [analytics.api.metaplow-proxy/throttler+ (throttle/make-throttler :metaplow-proxy :attempts-threshold 1)]+ ~@body))++(defmacro ^:private with-forward-capacity+ "Pin the relay's bulkhead to `n` concurrent forwards. `0` leaves no permit, so every request is shed."+ [n & body]+ `(with-redefs [analytics.api.metaplow-proxy/forward-bulkhead (dh.bh/bulkhead {:concurrency ~n})]+ ~@body))++(deftest sheds-forward-without-bulkhead-permit-test+ (testing "a request that cannot take a bulkhead permit is shed with a 503 and never reaches the collector"+ (with-forward-capacity 0+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [captured]+ (is (=? {:status 503 :body nil}+ (client/client-full-response :post 503 "metaplow-proxy/send" sample-payload)))+ (is (nil? @captured) "shedding must happen before the outbound call, not after it"))))))++(deftest releases-bulkhead-permit-after-forward-test+ (testing "each forward hands its permit back, so a single-permit bulkhead still serves serial requests"+ (with-forward-capacity 1+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [_captured]+ (dotimes [_ 3]+ (is (= {} (client/client :post 200 "metaplow-proxy/send" sample-payload)))))))))++(deftest throttles-anonymous-caller-test+ (testing "an anonymous caller is let through under the per-IP rate and rejected with a 429 above it"+ (with-one-request-budget+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [_captured]+ (let [opts (request-options "10.1.2.3")]+ (testing "permitted path"+ (is (= {} (client/client :post 200 "metaplow-proxy/send" opts sample-payload))))+ (testing "throttled path"+ (is (=? throttled-response+ (client/client-full-response :post 429 "metaplow-proxy/send" opts sample-payload))))))))))++(deftest throttle-is-keyed-per-source-address-test+ (testing "one caller exhausting its budget does not throttle a different source address"+ (with-one-request-budget+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [_captured]+ (is (= {} (client/client :post 200 "metaplow-proxy/send" (request-options "10.1.2.3") sample-payload)))+ (is (=? throttled-response+ (client/client-full-response :post 429 "metaplow-proxy/send" (request-options "10.1.2.3")+ sample-payload)))+ (is (= {} (client/client :post 200 "metaplow-proxy/send" (request-options "10.9.9.9")+ sample-payload))))))))++(deftest throttled-request-never-reaches-the-collector-test+ (testing "a throttled request is rejected before the handler forwards anything, so it holds no outbound connection"+ (with-one-request-budget+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [captured]+ (let [opts (request-options "10.1.2.3")]+ (client/client :post 200 "metaplow-proxy/send" opts sample-payload)+ (reset! captured nil)+ (client/client-full-response :post 429 "metaplow-proxy/send" opts sample-payload)+ (is (nil? @captured) "nothing should reach the collector")))))))++(deftest throttle-runs-before-body-validation-test+ (testing "a rejected payload still spends the caller's budget, so malformed floods cannot evade the throttle"+ (with-one-request-budget+ (with-collector! {:status 200 :headers {} :body "{}"}+ (fn [_captured]+ (let [opts (request-options "10.1.2.3")]+ (client/client :post 400 "metaplow-proxy/send" opts {:type "event"})+ (is (=? throttled-response+ (client/client-full-response :post 429 "metaplow-proxy/send" opts sample-payload)))))))))diff --git a/test/metabase/analytics/api/proxy_test.clj b/test/metabase/analytics/api/proxy_test.cljindex 45142836416..3adcbed0e3b 100644--- a/test/metabase/analytics/api/proxy_test.clj+++ b/test/metabase/analytics/api/proxy_test.clj@@ -6,10 +6,14 @@requests."(:require[clj-http.fake :as fake]+ [clojure.string :as str][clojure.test :refer :all]+ [diehard.bulkhead :as dh.bh]+ [metabase.analytics.api.proxy :as analytics.api.proxy][metabase.test :as mt][metabase.test.http-client :as client]- [metabase.util.json :as json]))+ [metabase.util.json :as json]+ [throttle.core :as throttle]))(set! *warn-on-reflection* true)@@ -41,6 +45,29 @@{(collector-url) (constantly ~response)}~@body)))))+(def ^:private throttled-response+ {:status 429+ :headers {"Retry-After" string?}+ :body {:error #(str/starts-with? % "Too many attempts!")}})++(defn- request-options+ "Request options that make `ip` the server-derived source address for the request."+ [ip]+ {:request-options {:headers {"x-forwarded-for" ip}}})++(defmacro ^:private with-one-request-budget+ "Pin the proxy throttler to a single request per source address, so one request exhausts a caller's budget."+ [& body]+ `(with-redefs [analytics.api.proxy/throttler+ (throttle/make-throttler :analytics-proxy :attempts-threshold 1)]+ ~@body))++(defmacro ^:private with-forward-capacity+ "Pin the relay's bulkhead to `n` concurrent forwards. `0` leaves no permit, so every request is shed."+ [n & body]+ `(with-redefs [analytics.api.proxy/forward-bulkhead (dh.bh/bulkhead {:concurrency ~n})]+ ~@body))+(deftest anonymous-public-route-test(testing "an anonymous POST reaches the handler (no auth required) and gets a 2xx back"(with-collector {:status 200 :headers {} :body "{}"}@@ -81,3 +108,59 @@(with-collector captured {:status 200 :headers {} :body "{}"}(client/client :post 200 "analytics-proxy" sample-payload))(is (nil? (get-in @captured [:headers "x-forwarded-for"]))))))++(deftest throttles-anonymous-caller-test+ (testing "an anonymous caller is let through under the per-IP rate and rejected with a 429 above it"+ (with-one-request-budget+ (with-collector {:status 200 :headers {} :body "{}"}+ (let [opts (request-options "10.1.2.3")]+ (testing "permitted path"+ (is (= {} (client/client :post 200 "analytics-proxy" opts sample-payload))))+ (testing "throttled path"+ (is (=? throttled-response+ (client/client-full-response :post 429 "analytics-proxy" opts sample-payload)))))))))++(deftest throttle-is-keyed-per-source-address-test+ (testing "one caller exhausting its budget does not throttle a different source address"+ (with-one-request-budget+ (with-collector {:status 200 :headers {} :body "{}"}+ (is (= {} (client/client :post 200 "analytics-proxy" (request-options "10.1.2.3") sample-payload)))+ (is (=? throttled-response+ (client/client-full-response :post 429 "analytics-proxy" (request-options "10.1.2.3")+ sample-payload)))+ (is (= {} (client/client :post 200 "analytics-proxy" (request-options "10.9.9.9") sample-payload)))))))++(deftest throttled-request-never-reaches-the-collector-test+ (testing "a throttled request is rejected before the handler forwards anything, so it holds no outbound connection"+ (let [captured (atom nil)]+ (with-collector captured {:status 200 :headers {} :body "{}"}+ (with-one-request-budget+ (let [opts (request-options "10.1.2.3")]+ (client/client :post 200 "analytics-proxy" opts sample-payload)+ (reset! captured nil)+ (client/client-full-response :post 429 "analytics-proxy" opts sample-payload)+ (is (nil? @captured))))))))++(deftest throttle-runs-before-body-validation-test+ (testing "a rejected payload still spends the caller's budget, so malformed floods cannot evade the throttle"+ (with-one-request-budget+ (with-collector {:status 200 :headers {} :body "{}"}+ (let [opts (request-options "10.1.2.3")]+ (client/client :post 400 "analytics-proxy" opts {:schema "" :data []})+ (is (=? throttled-response+ (client/client-full-response :post 429 "analytics-proxy" opts sample-payload))))))))+(deftest sheds-forward-without-bulkhead-permit-test+ (testing "a request that cannot take a bulkhead permit is shed with a 503 and never reaches the collector"+ (let [captured (atom nil)]+ (with-forward-capacity 0+ (with-collector captured {:status 200 :headers {} :body "{}"}+ (is (=? {:status 503}+ (client/client-full-response :post 503 "analytics-proxy" sample-payload)))))+ (is (nil? @captured) "shedding must happen before the outbound call, not after it"))))++(deftest releases-bulkhead-permit-after-forward-test+ (testing "each forward hands its permit back, so a single-permit bulkhead still serves serial requests"+ (with-forward-capacity 1+ (with-collector {:status 200 :headers {} :body "{}"}+ (dotimes [_ 3]+ (is (= {} (client/client :post 200 "analytics-proxy" sample-payload))))))))diff --git a/test/metabase/analytics/metaplow_test.clj b/test/metabase/analytics/metaplow_test.cljindex 66f12b2cf98..f0e2c7c6d43 100644--- a/test/metabase/analytics/metaplow_test.clj+++ b/test/metabase/analytics/metaplow_test.clj@@ -1,18 +1,51 @@(ns metabase.analytics.metaplow-test(:require+ [clj-http.client :as http][clojure.test :refer :all]+ [diehard.circuit-breaker :as dh.cb][metabase.analytics.metaplow :as metaplow][metabase.analytics.settings :as analytics.settings][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.util.json :as json]+ [metabase.util.retry :as retry][metabase.version.core :as version])(:import+ (dev.failsafe CircuitBreaker)+ (java.net ConnectException)(java.util.concurrent CountDownLatch TimeUnit)))(set! *warn-on-reflection* true)(use-fixtures :once (fixtures/initialize :db))+(defn- send-stub+ "Returns `[attempts stub]`, where `stub` stands in for [[metabase.analytics.metaplow/send-event!]] and `attempts`+ counts the calls to it. `stub` hands back `responses` in order, repeating the last once exhausted, after sleeping+ `sleep-ms` to stand in for a slow collector. After `limit` calls it switches to a non-retryable status, so a retry+ policy that has gone unbounded fails an assertion loudly instead of hanging the run."+ [responses & {:keys [limit sleep-ms] :or {limit 20 sleep-ms 0}}]+ (let [attempts (atom 0)+ responses (vec responses)]+ [attempts (fn [_payload]+ (let [n (swap! attempts inc)]+ (when (pos? sleep-ms)+ (Thread/sleep (long sleep-ms)))+ (if (> n limit)+ {:status 400}+ (nth responses (min (dec n) (dec (count responses)))))))]))++(defmacro ^:private with-fast-retries+ "Run `body` under the real retry policy with its backoff collapsed, so tests that exhaust retries don't sleep for+ seconds. The attempt count and the duration ceiling still come from the policy itself.++ Only for sends driven directly from the test thread. `pipeline-blocking` captures its workers' dynamic bindings+ once, when the first event of the JVM is enqueued, so wrapping a [[metabase.analytics.metaplow/track-event!]] call+ in this would pin the collapsed backoff for the rest of the run."+ [& body]+ `(binding [retry/*test-time-config-hook* #(assoc % :backoff-ms [1 2])]+ ~@body))+(deftest build-payload-test(testing "Schemas with an `:event` key produce `<schema>.<event>` and the event key is removed from data"(let [payload (#'metaplow/build-payload :snowplow/dashboard@@ -55,50 +88,195 @@(is (=? {:payload {:hostname "anonymous.metabase.com"}}(#'metaplow/build-payload :snowplow/dashboard {:event :dashboard-created}))))))+(defn- track-with-collecting-enqueue!+ "Calls `track-event!` with [[metabase.analytics.metaplow/enqueue!]] stubbed out, and returns+ `[returned-value enqueued-payloads]`."+ []+ (let [collector (atom [])]+ (mt/with-dynamic-fn-redefs [metaplow/enqueue! (fn [payload]+ (swap! collector conj payload)+ true)]+ [(metaplow/track-event! :snowplow/dashboard {:event :dashboard-created}) @collector])))+(deftest tracking-disabled-test(testing "When metaplow-tracking-enabled is false the event is not enqueued and the call returns false"- (mt/with-temporary-setting-values [metaplow-url nil]- (let [collector (atom [])]- (mt/with-dynamic-fn-redefs [metaplow/enqueue! (fn [payload]- (swap! collector conj payload)- true)]- (is (false? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created})))- (is (empty? @collector)))))))+ (testing "- no collector URL resolves"+ (mt/with-temporary-setting-values [anon-tracking-enabled true+ metaplow-url nil]+ (is (= [false []] (track-with-collecting-enqueue!)))))+ (testing "- anonymous tracking is off, even though a collector URL resolves"+ (mt/with-temporary-setting-values [anon-tracking-enabled false+ metaplow-url "http://fake-metaplow/api/send"]+ (is (= [false []] (track-with-collecting-enqueue!))))))+ (testing "With anonymous tracking on and a collector URL resolved, the event is enqueued"+ (mt/with-temporary-setting-values [anon-tracking-enabled true+ metaplow-url "http://fake-metaplow/api/send"]+ (let [[returned payloads] (track-with-collecting-enqueue!)]+ (is (true? returned))+ (is (=? [{:payload {:name "dashboard.dashboard_created"}}] payloads))))))++(deftest send-event!-test+ (mt/with-temporary-setting-values [metaplow-url "http://fake-metaplow/api/send"]+ (testing "The collector's response is returned as-is"+ (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts] {:status 200 :body "ok"})]+ (is (=? {:status 200} (#'metaplow/send-event! {:type "event"})))))+ (testing "A connection failure is caught and reported as status -1"+ (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts] (throw (ConnectException. "nope")))]+ (is (= {:status -1} (#'metaplow/send-event! {:type "event"})))))+ (testing "The payload is POSTed to `metaplow-url` as JSON"+ (let [calls (atom [])]+ (mt/with-dynamic-fn-redefs [http/post (fn [url opts] (swap! calls conj [url opts]) {:status 200})]+ (#'metaplow/send-event! {:type "event" :payload {:name "dashboard.created"}})+ (let [[[url opts]] @calls]+ (is (= "http://fake-metaplow/api/send" url))+ (is (= :json (:content-type opts)))+ (is (= {"type" "event" "payload" {"name" "dashboard.created"}}+ (json/decode (:body opts))))))))))++(deftest retry-policy-test+ (testing "The policy is explicit in code rather than inherited from the retry library's defaults"+ (is (=? {:max-retries 2+ :backoff-ms [500 2000]+ :max-duration-ms 5000}+ @#'metaplow/retry-policy)))+ ;; a breaker of its own, so breaker state neither leaks in from another test nor out of this one. Plain+ ;; `with-redefs`: the var holds a CircuitBreaker, which `mt/with-dynamic-fn-redefs` rejects as a non-fn+ (with-redefs [metaplow/breaker (#'metaplow/new-breaker nil)]+ (with-fast-retries+ (testing "A permanently failing collector is attempted a bounded number of times, then given up on"+ (let [[attempts stub] (send-stub [{:status -1}])]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]+ (is (= :error (#'metaplow/send-event-with-retries! {:type "event"})))+ (is (= 3 @attempts) "one initial attempt plus two retries, and no more"))))+ (testing "A collector that accepts the event is called exactly once"+ (let [[attempts stub] (send-stub [{:status 200}])]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]+ (is (= :sent (#'metaplow/send-event-with-retries! {:type "event"})))+ (is (= 1 @attempts)))))+ (testing "A transient failure is retried and the retry's success is reported as `:sent`"+ (let [[attempts stub] (send-stub [{:status 503} {:status 200}])]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]+ (is (= :sent (#'metaplow/send-event-with-retries! {:type "event"})))+ (is (= 2 @attempts)))))+ (testing "A status the collector will never accept is not retried"+ (doseq [status [400 401 403 410 422]]+ (let [[attempts stub] (send-stub [{:status status}])]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]+ (is (= :error (#'metaplow/send-event-with-retries! {:type "event"})))+ (is (= 1 @attempts) (str "status " status " should not be retried")))))))+ (testing "An attempt that outlasts the duration ceiling is not retried, even though retries remain"+ ;; the real ceiling is 5s and `send-event!`'s own timeouts are 5s apiece, so against a collector that drops+ ;; packets rather than refusing the connection a failing event gets one attempt, not three+ ;; `update ... min` rather than `assoc`, so this fails loudly if the policy ever loses its duration ceiling+ (binding [retry/*test-time-config-hook* #(-> (update % :max-duration-ms min 10)+ (assoc :backoff-ms [1 2]))]+ (let [[attempts stub] (send-stub [{:status -1}] :sleep-ms 50)]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]+ (is (= :error (#'metaplow/send-event-with-retries! {:type "event"})))+ (is (= 1 @attempts) "the duration ceiling stopped the retries")))))))++(deftest circuit-breaker-test+ (testing "The breaker policy is explicit in code: a failure threshold, a cooldown, and a success threshold"+ (is (=? {:failure-threshold 10+ :delay-ms 60000+ :success-threshold 1}+ @#'metaplow/breaker-config)))+ (with-fast-retries+ (testing "Ten consecutive failed events open the breaker, and further events are rejected without a send"+ (let [breaker (#'metaplow/new-breaker nil)+ [attempts stub] (send-stub [{:status -1}] :limit 200)]+ (with-redefs [metaplow/breaker breaker]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]+ (is (= (repeat 10 :error)+ (vec (repeatedly 10 #(#'metaplow/send-event-with-retries! {:type "event"}))))+ "the first ten events are attempted and fail")+ (is (= 30 @attempts) "ten events at three attempts each")+ (is (= :open (dh.cb/state breaker)))+ (reset! attempts 0)+ (is (= :rejected (#'metaplow/send-event-with-retries! {:type "event"})))+ (is (zero? @attempts) "no send is attempted while the breaker is open")))))+ (testing "A success closes the breaker again once it half-opens"+ (let [breaker (#'metaplow/new-breaker nil)]+ (with-redefs [metaplow/breaker breaker]+ (let [[_ fail] (send-stub [{:status -1}] :limit 200)]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! fail]+ (dotimes [_ 10] (#'metaplow/send-event-with-retries! {:type "event"}))+ (is (= :open (dh.cb/state breaker)))))+ ;; stand in for the 60-second cooldown elapsing, rather than racing a timer+ (.halfOpen ^CircuitBreaker breaker)+ (let [[attempts ok] (send-stub [{:status 200}])]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! ok]+ (is (= :sent (#'metaplow/send-event-with-retries! {:type "event"}))+ "the half-open probe is let through")+ (is (= 1 @attempts))+ (is (= :closed (dh.cb/state breaker)))))+ (testing "and a later failure is attempted rather than rejected"+ (let [[attempts fail] (send-stub [{:status -1}])]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! fail]+ (is (= :error (#'metaplow/send-event-with-retries! {:type "event"})))+ (is (= 3 @attempts))))))))))++(deftest error-metrics-test+ (mt/with-prometheus-system! [_ system]+ (let [errors (fn [stage]+ (or (mt/metric-value system :metabase-metaplow/errors {:stage stage}) 0.0))]+ (testing "A failed outbound attempt is counted against the send stage"+ (mt/with-temporary-setting-values [metaplow-url "http://fake-metaplow/api/send"]+ (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts] (throw (ConnectException. "nope")))]+ (#'metaplow/send-event! {:type "event"})))+ (is (= 1.0 (errors :send-event!))))+ (with-fast-retries+ (let [[_ fail] (send-stub [{:status -1}] :limit 200)]+ (with-redefs [metaplow/breaker (#'metaplow/new-breaker nil)]+ (mt/with-dynamic-fn-redefs [metaplow/send-event! fail]+ (testing "Giving up on an event is counted against the delivery stage"+ (#'metaplow/send-event-with-retries! {:type "event"})+ (is (= 1.0 (errors :send-event-with-retries!)))+ (is (= 0.0 (errors :circuit-breaker-open))))+ (testing "A breaker rejection is counted under its own stage, distinct from a delivery failure"+ (dotimes [_ 9] (#'metaplow/send-event-with-retries! {:type "event"}))+ (is (= 10.0 (errors :send-event-with-retries!)))+ (#'metaplow/send-event-with-retries! {:type "event"})+ (is (= 1.0 (errors :circuit-breaker-open)))+ (is (= 10.0 (errors :send-event-with-retries!))+ "a rejected event is not also counted as a delivery failure")))))))))(deftest pipeline-integration-test(mt/with-temporary-setting-values [metaplow-url "http://fake-metaplow/api/send"anon-tracking-enabled true]- (testing "track-event! enqueues onto the real channel and the pipeline worker invokes send-event-with-retries!"- (let [received (promise)]- ;; the pipeline's worker threads are spawned once from a defonce delay, so they never inherit a- ;; test's *local-redefs*; only a root swap reaches them- #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}- (with-redefs [metaplow/send-event-with-retries! (fn [payload]- (deliver received payload)- :sent)]- (is (true? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id 7})))- (let [payload (deref received 1000 ::timeout)]- (is (not= ::timeout payload)- "Worker did not consume the event within 1s")- (when (not= ::timeout payload)- (is (=? {:type "event"- :payload {:name "dashboard.dashboard_created"- :data {"dashboard_id" 7}}}- payload)))))))- (testing "Sending 200 events: all of them traverse the pipeline"- (let [received (atom [])- latch (CountDownLatch. 200)]- ;; the pipeline's worker threads are spawned once from a defonce delay, so they never inherit a- ;; test's *local-redefs*; only a root swap reaches them- #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}- (with-redefs [metaplow/send-event-with-retries! (fn [payload]- (swap! received conj payload)- (.countDown latch)- :sent)]- (doseq [i (range 1 201)]- (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id i}))- (is (true? (.await latch 1 TimeUnit/SECONDS))- "Pipeline did not process all 200 events within 1s")- (is (= 200 (count @received)))- (is (= (set (range 1 201))- (set (map #(get-in % [:payload :data "dashboard_id"]) @received)))))))))+ ;; a breaker of its own, so an unrelated test that left the process-wide one open can't make this one flake+ (with-redefs [metaplow/breaker (#'metaplow/new-breaker nil)]+ (testing "track-event! enqueues onto the real channel and a pipeline worker POSTs the payload"+ (let [received (promise)]+ ;; the pipeline's worker threads are spawned once from a defonce delay, so they never inherit a+ ;; test's *local-redefs*; only a root swap reaches them+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}+ (with-redefs [metaplow/send-event! (fn [payload]+ (deliver received payload)+ {:status 200})]+ (is (true? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id 7})))+ (let [payload (deref received 1000 ::timeout)]+ (is (not= ::timeout payload)+ "Worker did not consume the event within 1s")+ (when (not= ::timeout payload)+ (is (=? {:type "event"+ :payload {:name "dashboard.dashboard_created"+ :data {"dashboard_id" 7}}}+ payload)))))))+ (testing "Sending 200 events: all of them traverse the pipeline"+ (let [received (atom [])+ latch (CountDownLatch. 200)]+ ;; the pipeline's worker threads are spawned once from a defonce delay, so they never inherit a+ ;; test's *local-redefs*; only a root swap reaches them+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}+ (with-redefs [metaplow/send-event! (fn [payload]+ (swap! received conj payload)+ (.countDown latch)+ {:status 200})]+ (doseq [i (range 1 201)]+ (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id i}))+ (is (true? (.await latch 1 TimeUnit/SECONDS))+ "Pipeline did not process all 200 events within 1s")+ (is (= 200 (count @received)))+ (is (= (set (range 1 201))+ (set (map #(get-in % [:payload :data "dashboard_id"]) @received))))))))))diff --git a/test/metabase/analytics/settings_test.clj b/test/metabase/analytics/settings_test.cljindex add6e67abc3..0fadae83022 100644--- a/test/metabase/analytics/settings_test.clj+++ b/test/metabase/analytics/settings_test.clj@@ -2,6 +2,8 @@(:require[clojure.test :refer :all][metabase.analytics.settings :as analytics.settings]+ [metabase.config.core :as config]+ [metabase.premium-features.core :as premium-features][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util.date-2 :as u.date]@@ -46,3 +48,58 @@clojure.lang.ExceptionInfo#"Setting analytics-pii-retention-enabled is not enabled because feature :audit-app is not available"(analytics.settings/analytics-pii-retention-enabled! true)))))))++(deftest metaplow-url-default-test+ (testing "In production run mode the collector URL defaults to the Metabase Track collector"+ (with-redefs [config/is-prod? true]+ (is (= "https://product-analytics-ingestion.metabase.com/api/send"+ (#'analytics.settings/default-metaplow-url)))))+ (testing "In dev, test and e2e run modes there is no default, so those runs send nothing"+ (with-redefs [config/is-prod? false]+ (is (nil? (#'analytics.settings/default-metaplow-url))))))++(deftest metaplow-url-is-unset-outside-production-test+ (testing "There is no default in this run mode, so the setting resolves as unset"+ (is (nil? (analytics.settings/metaplow-url)))+ ;; a blank env var value is treated as absent, so this is also what a run with `MB_METAPLOW_URL=` resolves to+ (mt/with-temp-env-var-value! [mb-metaplow-url ""]+ (is (nil? (analytics.settings/metaplow-url)))+ (is (false? (analytics.settings/metaplow-tracking-enabled))))))++(deftest metaplow-url-air-gap-test+ (mt/with-temporary-setting-values [anon-tracking-enabled true+ metaplow-url nil]+ (with-redefs [config/is-prod? true]+ (testing "In production run mode with nothing set, the Metabase Track collector is the default"+ (mt/with-dynamic-fn-redefs [premium-features/airgap-enabled (constantly false)]+ (is (= "https://product-analytics-ingestion.metabase.com/api/send"+ (analytics.settings/metaplow-url)))+ (is (true? (analytics.settings/metaplow-tracking-enabled)))))+ (mt/with-dynamic-fn-redefs [premium-features/airgap-enabled (constantly true)]+ (testing "An air-gapped instance gets no default collector, so nothing is sent"+ (is (nil? (analytics.settings/metaplow-url)))+ (is (false? (analytics.settings/metaplow-tracking-enabled))))+ (testing "An air-gapped instance still sends to an explicitly set collector"+ (mt/with-temporary-setting-values [metaplow-url "https://collector.example.com/api/send"]+ (is (= "https://collector.example.com/api/send"+ (analytics.settings/metaplow-url)))+ (is (true? (analytics.settings/metaplow-tracking-enabled)))))+ (testing "...including one set by env var"+ (mt/with-temp-env-var-value! [mb-metaplow-url "https://env-collector.example.com/api/send"]+ (is (= "https://env-collector.example.com/api/send"+ (analytics.settings/metaplow-url)))+ (is (true? (analytics.settings/metaplow-tracking-enabled)))))))))++(deftest metaplow-tracking-enabled-test+ (testing "Metabase Track is on when anonymous tracking is on and a collector URL resolves"+ (mt/with-temporary-setting-values [anon-tracking-enabled true+ metaplow-url "https://collector.example.com/api/send"]+ (is (true? (analytics.settings/metaplow-tracking-enabled))))))++(deftest anon-tracking-opt-out-suppresses-both-destinations-test+ (testing "Turning anonymous tracking off suppresses Metabase Track as well as Snowplow"+ (mt/with-temporary-setting-values [anon-tracking-enabled false+ metaplow-url "https://collector.example.com/api/send"+ snowplow-available true]+ (is (false? (analytics.settings/metaplow-tracking-enabled)))+ (is (false? (analytics.settings/snowplow-enabled))))))diff --git a/test/metabase/analytics/snowplow_test.clj b/test/metabase/analytics/snowplow_test.cljindex 4737576c362..8bf71c1ea71 100644--- a/test/metabase/analytics/snowplow_test.clj+++ b/test/metabase/analytics/snowplow_test.clj@@ -14,8 +14,11 @@[toucan2.core :as t2])(:import(com.snowplowanalytics.snowplow.tracker.events SelfDescribing)+ (com.snowplowanalytics.snowplow.tracker.http ApacheHttpClientAdapter)(com.snowplowanalytics.snowplow.tracker.payload SelfDescribingJson)- (java.util LinkedHashMap)))+ (java.net InetAddress ServerSocket)+ (java.util LinkedHashMap)+ (org.apache.http.client.config CookieSpecs RequestConfig)))(set! *warn-on-reflection* true)@@ -185,3 +188,30 @@(mt/with-temporary-setting-values [anon-tracking-enabled false](analytics/track-event! :snowplow/account {:event :new_instance_created} nil)(is (= [] (pop-event-data-and-user-id!))))))))))++(deftest ^:parallel http-timeouts-test+ (testing "the Snowplow HTTP client has finite timeouts"+ (let [^RequestConfig config (#'snowplow/request-config)]+ ;; Apache HttpClient reads any non-positive value as "wait forever".+ (is (pos? (.getConnectTimeout config))+ "connect timeout must be finite")+ (is (pos? (.getSocketTimeout config))+ "socket timeout must be finite")+ (is (pos? (.getConnectionRequestTimeout config))+ "connection-request timeout must be finite")+ (testing "and still sets the cookie policy (PR #24579)"+ (is (= CookieSpecs/STANDARD (.getCookieSpec config)))))))++(deftest ^:parallel unresponsive-collector-test+ (testing "a collector that accepts connections but never responds cannot hold an emitter thread indefinitely"+ ;; A `ServerSocket` nobody ever calls `.accept` on still completes the TCP handshake out of its backlog, so this+ ;; is a connected socket that will never produce a response.+ (with-open [blackhole (ServerSocket. 0 1 (InetAddress/getLoopbackAddress))]+ (let [^ApacheHttpClientAdapter adapter (#'snowplow/http-client-adapter+ (str "http://127.0.0.1:" (.getLocalPort blackhole)))+ event (SelfDescribingJson. "iglu:com.metabase/instance/jsonschema/1-1-2" {})+ response (future (.post adapter event))+ result (deref response 15000 ::never-returned)]+ (future-cancel response)+ (is (= -1 result)+ "the POST should report a failed request rather than block forever")))))diff --git a/test/metabase/server/middleware/security_test.clj b/test/metabase/server/middleware/security_test.cljindex c9e4c439533..8b846fe1511 100644--- a/test/metabase/server/middleware/security_test.clj+++ b/test/metabase/server/middleware/security_test.clj@@ -89,6 +89,22 @@(testing "and script-src-elem specifically is never introduced"(is (not (contains? (csp-directive-names injection) "script-src-elem")))))))+(deftest csp-header-analytics-collectors-test+ (testing "the analytics collectors are in connect-src so the browser legs can post to them"+ (mt/with-temporary-setting-values [anon-tracking-enabled true+ metaplow-url "https://product-analytics-ingestion.metabase.com/api/send"+ snowplow-url "https://sp.metabase.com"]+ (let [connect-src (csp-directive "connect-src")]+ (is (str/includes? connect-src "https://product-analytics-ingestion.metabase.com/api/send"))+ (is (str/includes? connect-src "https://sp.metabase.com")))))+ (testing "and neither is listed when anonymous tracking is off"+ (mt/with-temporary-setting-values [anon-tracking-enabled false+ metaplow-url "https://product-analytics-ingestion.metabase.com/api/send"+ snowplow-url "https://sp.metabase.com"]+ (let [connect-src (csp-directive "connect-src")]+ (is (not (str/includes? connect-src "product-analytics-ingestion.metabase.com")))+ (is (not (str/includes? connect-src "sp.metabase.com")))))))+(deftest csp-header-iframe-hosts-tests(testing "Allowed iframe hosts setting is used in the CSP frame-src directive."(mt/with-temporary-setting-values [allowed-iframe-hosts "https://www.wikipedia.org, https://www.typescriptlang.org/ https://clojure.org"]