Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GRO-1083: Send analytics to Metabase Track by default on self-hosted and OSS instances (#84252)

John Swanson57006c684fc3
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 1cab8dd4fc7..0768922d880 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -180,6 +180,7 @@
:module-exports #{analytics.interface
analytics.internal-stats}
:api #{metabase.analytics.api
+ metabase.analytics.api.metaplow-proxy
metabase.analytics.api.proxy
metabase.analytics.core
metabase.analytics.event
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 632e737b8ab..b84199e8940 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -28,7 +28,7 @@
:metabase/discourage-millis-duration 1
:metabase/modules 23
:metabase/namespace-name 3
- :metabase/validate-defendpoint-has-response-schema 426
+ :metabase/validate-defendpoint-has-response-schema 427
:metabase/validate-defendpoint-query-params-use-kebab-case 48
:metabase/validate-defendpoint-route-uses-kebab-case 44
:missing-docstring 4
diff --git a/.github/actions/e2e-prepare-containers/action.yml b/.github/actions/e2e-prepare-containers/action.yml
index 0c641701127..927c4d749dd 100644
--- a/.github/actions/e2e-prepare-containers/action.yml
+++ b/.github/actions/e2e-prepare-containers/action.yml
@@ -39,6 +39,10 @@ inputs:
description: Snowplow micro container
required: false
default: false
+ metaplow:
+ description: Metabase Track recording sink container
+ required: false
+ default: false
runs:
using: "composite"
@@ -80,6 +84,7 @@ runs:
[ "${{ inputs.mocksaml }}" = true ] && SERVICES+=(saml-idp)
[ "${{ inputs.webhook }}" = true ] && SERVICES+=(webhook-tester)
[ "${{ inputs.snowplow }}" = true ] && SERVICES+=(snowplow-micro)
+ [ "${{ inputs.metaplow }}" = true ] && SERVICES+=(metaplow-sink)
for attempt in 1 2 3; do
echo "Docker compose up attempt $attempt of 3..."
@@ -149,3 +154,9 @@ runs:
while ! nc -z localhost 9090; do sleep 1; done &&
echo -e "${G}Snowplow micro is up and running!"
fi
+
+ if ${{ inputs.metaplow }}; then
+ echo -e "${Y}checking Metabase Track sink container..." &&
+ while ! nc -z localhost 9091; do sleep 1; done &&
+ echo -e "${G}Metabase Track sink is up and running!"
+ fi
diff --git a/.github/file-paths.yaml b/.github/file-paths.yaml
index bf85f91ba48..995e1e62898 100644
--- a/.github/file-paths.yaml
+++ b/.github/file-paths.yaml
@@ -67,6 +67,7 @@ frontend_specs: &frontend_specs
- "frontend/**/tests/**"
- "frontend/**/*.unit.*"
- "enterprise/frontend/**/*.unit.*"
+ - "metaplow/**/*.unit.*"
- "jest.config.js"
- "jest.base.conf.js"
- "jest.esm-packages.js"
@@ -126,8 +127,12 @@ ci_scripts:
- ".github/file-paths.yaml"
- ".github/scripts/**"
- ".github/actions/**"
- - ".github/workflows/e2e-{test,tests,matrix-builder}.yml"
- - ".github/workflows/loki.yml"
+ # e2e-containers.unit.spec.ts asserts that every workflow starting one analytics collector
+ # container starts the other, and that three files agree on the sink's port. A new workflow is
+ # exactly the drift it exists to catch, so this follows all of them, not a list.
+ - ".github/workflows/*.{yml,yaml}"
+ - "metaplow/**"
+ - "deps.edn"
- ".storybook/story-files.cjs"
- "e2e/runner/read-spec-paths.js"
- "e2e/support/cypress.config.js"
@@ -270,6 +275,11 @@ snowplow: &snowplow
- "bin/lint-snowplow-schemas.bb"
- "bin/lint-snowplow-schemas-test.bb"
+# The recording sink the e2e suite asserts Metabase Track events against, the counterpart to
+# `snowplow/` above.
+metaplow: &metaplow
+ - "metaplow/**"
+
# e2e harness/support code (commands, support file, runner). Not in the
# instrumented bundle, so it maps to no covered module — a change here forces a
# full e2e run (see create-test-plan / e2eInfraTouched).
@@ -279,6 +289,7 @@ e2e_infra: &e2e_infra
- *frontend_configs
- *shared_e2e
- *snowplow
+ - *metaplow
- "e2e/**/!(*.cy.spec.@(js|jsx|ts|tsx))"
- "frontend/build/**"
- "patches/**"
diff --git a/.github/scripts/e2e-containers.unit.spec.ts b/.github/scripts/e2e-containers.unit.spec.ts
new file mode 100644
index 00000000000..3e2575c7bb7
--- /dev/null
+++ b/.github/scripts/e2e-containers.unit.spec.ts
@@ -0,0 +1,74 @@
+import { readFileSync, readdirSync } from "fs";
+import { join } from "path";
+
+import { load } from "js-yaml";
+
+const GITHUB = join(__dirname, "..");
+const WORKFLOWS = join(GITHUB, "workflows");
+const ROOT = join(GITHUB, "..");
+
+const PREPARE_CONTAINERS = "./.github/actions/e2e-prepare-containers";
+
+type Step = { uses?: string; with?: Record<string, unknown> };
+type Workflow = { jobs?: Record<string, { steps?: Step[] }> };
+
+/** Every `e2e-prepare-containers` step across the workflows, labelled with the job it lives in. */
+const prepareContainerSteps = () =>
+ readdirSync(WORKFLOWS)
+ .filter((file) => file.endsWith(".yml") || file.endsWith(".yaml"))
+ .flatMap((file) => {
+ // `js-yaml` returns `unknown`; the shape we read back is the slice of the workflow schema
+ // declared above, and a workflow that does not match it simply contributes no steps.
+ const workflow = load(
+ readFileSync(join(WORKFLOWS, file), "utf8"),
+ ) as Workflow;
+
+ return Object.entries(workflow.jobs ?? {}).flatMap(([job, { steps }]) =>
+ (steps ?? [])
+ .filter((step) => step.uses === PREPARE_CONTAINERS)
+ .map((step) => ({
+ where: `${file} / ${job}`,
+ with: step.with ?? {},
+ })),
+ );
+ });
+
+const read = (...path: string[]) => readFileSync(join(ROOT, ...path), "utf8");
+
+describe("e2e container setup", () => {
+ it("finds the prepare-containers steps it is asserting over", () => {
+ expect(prepareContainerSteps().length).toBeGreaterThan(0);
+ });
+
+ // The `:e2e` alias in deps.edn points the backend at both analytics collectors unconditionally,
+ // so a job that starts one has to start the other. Letting them drift apart leaves a job posting
+ // every event at a port nothing is listening on — bounded, thanks to the sender's circuit
+ // breaker, but a confusing thing to find in a log.
+ it("starts the Metabase Track sink wherever it starts Snowplow micro", () => {
+ const missing = prepareContainerSteps()
+ .filter(
+ (step) => step.with.snowplow === true && step.with.metaplow !== true,
+ )
+ .map((step) => step.where);
+
+ expect(missing).toEqual([]);
+ });
+
+ // Three files have to agree on the sink's port and nothing else ties them together: the alias
+ // the backend is started with, the port the container publishes, and the readiness check CI
+ // blocks on. The e2e helper hardcodes it a fourth time, but that one is covered by the specs.
+ it("agrees on the Metabase Track sink's port", () => {
+ const [, port] =
+ read("deps.edn").match(
+ /"-Dmb\.metaplow\.url=http:\/\/localhost:(\d+)\//,
+ ) ?? [];
+
+ expect(port).toBeDefined();
+ expect(read("metaplow", "docker-compose.yml")).toContain(
+ `"${port}:${port}"`,
+ );
+ expect(
+ read(".github", "actions", "e2e-prepare-containers", "action.yml"),
+ ).toContain(`nc -z localhost ${port}`);
+ });
+});
diff --git a/.github/scripts/file-paths.unit.spec.ts b/.github/scripts/file-paths.unit.spec.ts
index 1ce38458c86..07867edc528 100644
--- a/.github/scripts/file-paths.unit.spec.ts
+++ b/.github/scripts/file-paths.unit.spec.ts
@@ -181,6 +181,26 @@ describe("file-paths.yaml", () => {
).toBe(false);
});
+ // The e2e Metabase Track sink lives outside frontend/ and e2e/, so nothing else would pull it
+ // into either gate: without these entries a sink-only change runs no tests at all. The e2e gate
+ // reaches it through e2e_infra, so a sink-only change also forces a full e2e run.
+ it("covers the e2e Metabase Track sink in both the e2e and the frontend unit gates", () => {
+ expect(matches("e2e_all", "metaplow/sink.js")).toBe(true);
+ expect(matches("e2e_infra", "metaplow/sink.js")).toBe(true);
+ expect(matches("frontend_specs", "metaplow/sink.unit.spec.js")).toBe(true);
+ });
+
+ // e2e-containers.unit.spec.ts asserts over these, and it runs in the ci-scripts suite.
+ it.each([
+ ".github/workflows/e2e-test.yml",
+ ".github/workflows/embedding-sdk-component-tests.yml",
+ ".github/actions/e2e-prepare-containers/action.yml",
+ "metaplow/docker-compose.yml",
+ "deps.edn",
+ ])("runs the CI script tests for %s, which a spec asserts over", (path) => {
+ expect(matches("ci_scripts", path)).toBe(true);
+ });
+
it("runs the ratchet check on the ratchets file", () => {
expect(matches("project_ratchet_checks", ".clj-kondo/ratchets.edn")).toBe(
true,
diff --git a/.github/workflows/e2e-coverage-shard.yml b/.github/workflows/e2e-coverage-shard.yml
index 2858a89ba7d..1edd6d02292 100644
--- a/.github/workflows/e2e-coverage-shard.yml
+++ b/.github/workflows/e2e-coverage-shard.yml
@@ -51,6 +51,7 @@ jobs:
openldap: true
webhook: true
snowplow: true
+ metaplow: true
postgres: true
mysql: true
mongo: true
diff --git a/.github/workflows/e2e-stress-test-flake-fix.yml b/.github/workflows/e2e-stress-test-flake-fix.yml
index 0f810905e7b..399e7027a59 100644
--- a/.github/workflows/e2e-stress-test-flake-fix.yml
+++ b/.github/workflows/e2e-stress-test-flake-fix.yml
@@ -167,6 +167,7 @@ jobs:
maildev: true
openldap: true
snowplow: true
+ metaplow: true
postgres: ${{ inputs.qa_db }}
mysql: ${{ inputs.qa_db }}
mongo: ${{ inputs.qa_db }}
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index f857247e8b3..439a1003989 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -107,6 +107,7 @@ jobs:
openldap: true
webhook: true
snowplow: true
+ metaplow: true
postgres: true
mysql: true
mongo: true
diff --git a/.github/workflows/embedding-sdk-component-tests.yml b/.github/workflows/embedding-sdk-component-tests.yml
index f31d0538987..2d8183748cd 100644
--- a/.github/workflows/embedding-sdk-component-tests.yml
+++ b/.github/workflows/embedding-sdk-component-tests.yml
@@ -49,6 +49,7 @@ jobs:
openldap: false
webhook: false
snowplow: true
+ metaplow: true
postgres: false
mysql: false
mongo: false
@@ -264,6 +265,7 @@ jobs:
openldap: false
webhook: false
snowplow: true
+ metaplow: true
postgres: false
mysql: false
mongo: false
diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
index fda9cb0beae..bc1a06b2d7d 100644
--- a/.github/workflows/pre-release.yml
+++ b/.github/workflows/pre-release.yml
@@ -184,6 +184,7 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }}
maildev: true
snowplow: true
+ metaplow: true
- name: Prepare front-end environment
uses: ./.github/actions/prepare-frontend
- name: Prepare back-end environment
diff --git a/deps.edn b/deps.edn
index 5a2afa00ac4..45aa3598912 100644
--- a/deps.edn
+++ b/deps.edn
@@ -415,6 +415,9 @@
;; We use Snowplow micro in e2e tests
"-Dmb.snowplow.available=true"
"-Dmb.snowplow.url=http://localhost:9090"
+ ;; ...and the recording sink in `metaplow/` for Metabase Track. `metaplow-url`'s production
+ ;; default is nil outside production run mode, so e2e has to name the collector itself.
+ "-Dmb.metaplow.url=http://localhost:9091/api/send"
;; See: https://github.com/metabase/metabase/pull/38280
"-Dmb.last.analytics.checksum=-1"
"-Dmb.experimental.search.block.on.queue=true"
diff --git a/e2e/support/helpers/e2e-deep-match.js b/e2e/support/helpers/e2e-deep-match.js
new file mode 100644
index 00000000000..402af0447c1
--- /dev/null
+++ b/e2e/support/helpers/e2e-deep-match.js
@@ -0,0 +1,46 @@
+export function isDeepMatch(objectOrValue, partialObjectOrValue) {
+ if (isMatcher(partialObjectOrValue)) {
+ return partialObjectOrValue(objectOrValue);
+ }
+
+ const bothAreNotObjects =
+ // Check null because typeof null === "object"
+ objectOrValue == null ||
+ partialObjectOrValue == null ||
+ typeof objectOrValue !== "object" ||
+ typeof partialObjectOrValue !== "object";
+
+ // Exit condition when calling recursively
+ if (bothAreNotObjects) {
+ return objectOrValue === partialObjectOrValue;
+ }
+
+ for (const [key, value] of Object.entries(partialObjectOrValue)) {
+ if (Array.isArray(value)) {
+ if (
+ !Array.isArray(objectOrValue[key]) ||
+ !isArrayDeepMatch(objectOrValue[key], value)
+ ) {
+ return false;
+ }
+ } else if (!isDeepMatch(objectOrValue[key], value)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+function isMatcher(value) {
+ return typeof value === "function";
+}
+
+function isArrayDeepMatch(array, partialArray) {
+ for (const index in partialArray) {
+ if (!isDeepMatch(array[index], partialArray[index])) {
+ return false;
+ }
+ }
+
+ return true;
+}
diff --git a/e2e/support/helpers/e2e-metaplow-helpers.ts b/e2e/support/helpers/e2e-metaplow-helpers.ts
new file mode 100644
index 00000000000..cea52854e70
--- /dev/null
+++ b/e2e/support/helpers/e2e-metaplow-helpers.ts
@@ -0,0 +1,59 @@
+import { isDeepMatch } from "./e2e-deep-match";
+
+type DeepMatch = Record<string, unknown>;
+
+const METAPLOW_SINK_URL = "http://localhost:9091";
+const METAPLOW_INTERVAL = 250;
+const METAPLOW_TIMEOUT = 5000;
+
+export interface MetaplowSinkEvent {
+ receivedAt: string;
+ path: string;
+ body: {
+ type?: string;
+ payload?: Record<string, unknown>;
+ } | null;
+ raw?: string;
+}
+
+export const resetMetaplow = () => {
+ sendMetaplowRequest<{ reset: true }>("sink/reset");
+};
+
+export const expectMetaplowRequest = (match: DeepMatch, count = 1) => {
+ retryForEvents(
+ (events) => events.filter((e) => isDeepMatch(e, match)).length === count,
+ ).should("be.ok");
+};
+
+export const expectMetaplowEvent = (payload: DeepMatch, count = 1) => {
+ expectMetaplowRequest({ body: { type: "event", payload } }, count);
+};
+
+const sendMetaplowRequest = <T>(url: string) => {
+ cy.log("Send a Metabase Track sink request");
+ return cy.request<T>({ url: `${METAPLOW_SINK_URL}/${url}`, json: true });
+};
+
+type EventsResponse = Cypress.Chainable<Cypress.Response<MetaplowSinkEvent[]>>;
+
+const retryForEvents = (
+ condition: (events: MetaplowSinkEvent[]) => boolean,
+ timeout = METAPLOW_TIMEOUT,
+): EventsResponse => {
+ return sendMetaplowRequest<MetaplowSinkEvent[]>("sink/events").then(
+ (response) => {
+ if (condition(response.body)) {
+ return cy.wrap(response);
+ }
+ if (timeout <= 0) {
+ const names = response.body.map((e) => e?.body?.payload?.name);
+ throw new Error(
+ `Metabase Track sink retry timeout. Events recorded: ${JSON.stringify(names).slice(0, 512)}`,
+ );
+ }
+ cy.wait(METAPLOW_INTERVAL);
+ return retryForEvents(condition, timeout - METAPLOW_INTERVAL);
+ },
+ );
+};
diff --git a/e2e/support/helpers/e2e-snowplow-helpers.js b/e2e/support/helpers/e2e-snowplow-helpers.js
index 3ef09279315..51e623cd7a1 100644
--- a/e2e/support/helpers/e2e-snowplow-helpers.js
+++ b/e2e/support/helpers/e2e-snowplow-helpers.js
@@ -1,5 +1,9 @@
import { updateSetting } from "e2e/support/helpers";
+import { isDeepMatch } from "./e2e-deep-match";
+
+export { isDeepMatch };
+
const SNOWPLOW_URL = "http://localhost:9090";
const SNOWPLOW_INTERVAL = 100;
// Snowplow events are POSTed asynchronously and then ingested by snowplow-micro, so a
@@ -54,53 +58,6 @@ export const expectUnstructuredSnowplowEvent = (eventData, count = 1) => {
);
};
-export function isDeepMatch(objectOrValue, partialObjectOrValue) {
- if (isMatcher(partialObjectOrValue)) {
- return partialObjectOrValue(objectOrValue);
- }
-
- const bothAreNotObjects =
- // Check null because typeof null === "object"
- objectOrValue == null ||
- partialObjectOrValue == null ||
- typeof objectOrValue !== "object" ||
- typeof partialObjectOrValue !== "object";
-
- // Exit condition when calling recursively
- if (bothAreNotObjects) {
- return objectOrValue === partialObjectOrValue;
- }
-
- for (const [key, value] of Object.entries(partialObjectOrValue)) {
- if (Array.isArray(value)) {
- if (
- !Array.isArray(objectOrValue[key]) ||
- !isArrayDeepMatch(objectOrValue[key], value)
- ) {
- return false;
- }
- } else if (!isDeepMatch(objectOrValue[key], value)) {
- return false;
- }
- }
-
- return true;
-}
-
-function isMatcher(value) {
- return typeof value === "function";
-}
-
-function isArrayDeepMatch(array, partialArray) {
- for (const index in partialArray) {
- if (!isDeepMatch(array[index], partialArray[index])) {
- return false;
- }
- }
-
- return true;
-}
-
export const expectNoBadSnowplowEvents = () => {
sendSnowplowRequest("micro/bad").its("body").should("deep.equal", []);
};
diff --git a/e2e/support/helpers/index.ts b/e2e/support/helpers/index.ts
index 66381e21bbe..0426b42c5ce 100644
--- a/e2e/support/helpers/index.ts
+++ b/e2e/support/helpers/index.ts
@@ -39,6 +39,7 @@ export * from "./e2e-explorations-helpers";
export * from "./e2e-filter-helpers";
export * from "./e2e-ldap-helpers";
export * from "./e2e-metabot-helpers";
+export * from "./e2e-metaplow-helpers";
export * from "./e2e-metric-page-helpers";
export * from "./e2e-metrics-viewer-helpers";
export * from "./e2e-misc-helpers";
diff --git a/e2e/test/scenarios/analytics/metabase-track.cy.spec.ts b/e2e/test/scenarios/analytics/metabase-track.cy.spec.ts
new file mode 100644
index 00000000000..74a8f7462d5
--- /dev/null
+++ b/e2e/test/scenarios/analytics/metabase-track.cy.spec.ts
@@ -0,0 +1,39 @@
+const { H } = cy;
+
+describe("scenarios > analytics > Metabase Track", () => {
+ beforeEach(() => {
+ H.restore();
+ cy.signInAsAdmin();
+ H.enableTracking();
+ H.resetMetaplow();
+ });
+
+ it("delivers a browser-originated event to the collector", () => {
+ cy.visit("/");
+ cy.findByTestId("home-page").should("be.visible");
+
+ H.expectMetaplowRequest({
+ path: "/api/send",
+ body: {
+ type: "event",
+ payload: {
+ name: "pageview",
+ url: "http://anonymous.metabase.com/",
+ hostname: "anonymous.metabase.com",
+ tag: "metabase-instance",
+ },
+ },
+ });
+ });
+
+ it("delivers a backend-originated event to the collector", () => {
+ H.createDashboard({ name: "Tracked dashboard" }).then(({ body }) => {
+ H.expectMetaplowEvent({
+ name: "dashboard.dashboard_created",
+ hostname: "anonymous.metabase.com",
+ tag: "metabase-instance",
+ data: { dashboard_id: body.id },
+ });
+ });
+ });
+});
diff --git a/e2e/test/scenarios/docker-compose.yml b/e2e/test/scenarios/docker-compose.yml
index 7e8e1fc0c19..9c74f2b1aaf 100644
--- a/e2e/test/scenarios/docker-compose.yml
+++ b/e2e/test/scenarios/docker-compose.yml
@@ -2,6 +2,7 @@ name: metabase-e2e
include:
- ../../../snowplow/docker-compose.yml
+ - ../../../metaplow/docker-compose.yml
services:
postgres-sample:
diff --git a/frontend/lint/config.mjs b/frontend/lint/config.mjs
index 96d3f691d53..bdf8ee21159 100644
--- a/frontend/lint/config.mjs
+++ b/frontend/lint/config.mjs
@@ -547,6 +547,19 @@ const configs = [
"no-console": "off",
},
},
+ {
+ files: ["metaplow/**/*.js"],
+ languageOptions: {
+ sourceType: "commonjs",
+ globals: {
+ ...globals.node,
+ },
+ },
+ rules: {
+ "no-console": "off",
+ "import/no-commonjs": "off",
+ },
+ },
{
files: ["frontend/src/metabase/**/*"],
diff --git a/frontend/lint/scripts/run-oxlint.mjs b/frontend/lint/scripts/run-oxlint.mjs
index bf9261bea56..5dea14cc161 100644
--- a/frontend/lint/scripts/run-oxlint.mjs
+++ b/frontend/lint/scripts/run-oxlint.mjs
@@ -9,7 +9,7 @@ const FLAGS = [
"0",
"--report-unused-disable-directives",
];
-const DEFAULT_PATHS = ["enterprise/frontend", "frontend", "e2e"];
+const DEFAULT_PATHS = ["enterprise/frontend", "frontend", "e2e", "metaplow"];
export function getOxlintArgs(args, pathExists) {
const hasPath = args.some(
diff --git a/frontend/lint/tests/oxlint-run.test.mjs b/frontend/lint/tests/oxlint-run.test.mjs
index cc5956408cc..f9b74e7bd69 100644
--- a/frontend/lint/tests/oxlint-run.test.mjs
+++ b/frontend/lint/tests/oxlint-run.test.mjs
@@ -9,7 +9,7 @@ const flags = [
"0",
"--report-unused-disable-directives",
];
-const defaultPaths = ["enterprise/frontend", "frontend", "e2e"];
+const defaultPaths = ["enterprise/frontend", "frontend", "e2e", "metaplow"];
const file = "frontend/src/metabase/dev.ts";
const pathExists = (arg) => arg === file;
diff --git a/frontend/src/embedding-sdk-bundle/analytics/tracker.ts b/frontend/src/embedding-sdk-bundle/analytics/tracker.ts
index 4b023a1fc48..c5d1aba8c54 100644
--- a/frontend/src/embedding-sdk-bundle/analytics/tracker.ts
+++ b/frontend/src/embedding-sdk-bundle/analytics/tracker.ts
@@ -15,8 +15,20 @@ import type { SdkStore } from "embedding-sdk-bundle/store/types";
import { getSdkPackageVersion } from "embedding-sdk-shared/lib/get-build-info";
import type { MetabaseAuthConfig } from "embedding-sdk-shared/types/auth-config";
import { isEmbeddingEajs } from "metabase/embedding-sdk/config";
+import { getSettings } from "metabase/settings";
import { initMetaplow } from "metabase/utils/metaplow";
+const METAPLOW_RELAY_PATH = "/api/metaplow-proxy";
+
+function getMetaplowHostUrl(store: SdkStore, metabaseInstanceUrl: string) {
+ const isEnabled = !!getSettings(store.getState())?.[
+ "metaplow-tracking-enabled"
+ ];
+ return isEnabled
+ ? `${metabaseInstanceUrl.replace(/\/+$/, "")}${METAPLOW_RELAY_PATH}`
+ : null;
+}
+
export function deriveAuthMethod(
authConfig: MetabaseAuthConfig,
): SdkAuthMethod {
@@ -44,6 +56,7 @@ export function useInitSdkTracker(
return;
}
initMetaplow({
+ hostUrl: getMetaplowHostUrl(reduxStore, authConfig.metabaseInstanceUrl),
// Via frontend/src/embedding-sdk-bundle/analytics/snowplow.ts
// Omits userId — unlike the main-app tracker, SDK component usage is tracked at instance granularity;
// the analytics-uuid already identifies the account.
diff --git a/frontend/src/embedding-sdk-bundle/analytics/tracker.unit.spec.tsx b/frontend/src/embedding-sdk-bundle/analytics/tracker.unit.spec.tsx
index 262300e5867..bbb92cd2ea7 100644
--- a/frontend/src/embedding-sdk-bundle/analytics/tracker.unit.spec.tsx
+++ b/frontend/src/embedding-sdk-bundle/analytics/tracker.unit.spec.tsx
@@ -16,16 +16,24 @@ jest.mock("embedding-sdk-shared/lib/get-build-info", () => ({
getSdkPackageVersion: jest.fn(() => "1.2.3"),
}));
+import { reset } from "@metabase/track";
import { renderHook } from "@testing-library/react";
+import { createMockState, seedApiQueryCache } from "__support__/state";
import { useIsTrackingEnabled } from "embedding-sdk-bundle/analytics/component-events";
import {
initSdkTracker,
trackSdkSimpleEvent,
} from "embedding-sdk-bundle/analytics/snowplow";
import { setSdkTrackerReady } from "embedding-sdk-bundle/store/reducer";
+import type { SdkStore } from "embedding-sdk-bundle/store/types";
+import { createMockSdkState } from "embedding-sdk-bundle/test/mocks/state";
import type { MetabaseAuthConfig } from "embedding-sdk-shared/types/auth-config";
import { EMBEDDING_SDK_IFRAME_EMBEDDING_CONFIG } from "metabase/embedding-sdk/config";
+import { trackMetaplowEvent } from "metabase/utils/metaplow";
+import Settings from "metabase/utils/settings";
+import type { EnterpriseSettings } from "metabase-types/api";
+import { createMockSettings } from "metabase-types/api/mocks";
import { deriveAuthMethod, useInitSdkTracker } from "./tracker";
@@ -33,9 +41,22 @@ const mockUseIsTrackingEnabled = jest.mocked(useIsTrackingEnabled);
const mockInitSdkTracker = jest.mocked(initSdkTracker);
const mockTrackSdkSimpleEvent = jest.mocked(trackSdkSimpleEvent);
-function makeStore() {
- // Unjustified type cast. FIXME
- return { dispatch: jest.fn(), getState: jest.fn() } as any;
+function makeStore(settings: Partial<EnterpriseSettings> = {}) {
+ const state = createMockState({ sdk: createMockSdkState() });
+ const stateWithSettings = {
+ ...state,
+ "metabase-api": seedApiQueryCache(state["metabase-api"], [
+ {
+ endpointName: "getSessionProperties",
+ value: createMockSettings(settings),
+ },
+ ]),
+ };
+ // The hook only dispatches and reads state; a full SdkStore is not needed.
+ return {
+ dispatch: jest.fn(),
+ getState: () => stateWithSettings,
+ } as unknown as SdkStore;
}
const SSO_AUTH_CONFIG: MetabaseAuthConfig = {
@@ -141,3 +162,83 @@ describe("useInitSdkTracker", () => {
expect(mockTrackSdkSimpleEvent).not.toHaveBeenCalled();
});
});
+
+describe("Metabase Track host URL", () => {
+ const COLLECTOR_URL = "https://collector.example.com/api/send";
+ const RELAY_SEND_URL = "https://metabase.example.com/api/metaplow-proxy/send";
+ const trackingOn = { "metaplow-tracking-enabled": true };
+
+ let fetchSpy: jest.SpyInstance;
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ mockUseIsTrackingEnabled.mockReturnValue(true);
+ mockInitSdkTracker.mockReturnValue(true);
+ fetchSpy = jest
+ .spyOn(global, "fetch")
+ .mockResolvedValue(new Response(null, { status: 200 }));
+ });
+
+ afterEach(() => {
+ fetchSpy.mockRestore();
+ Settings.set("metaplow-url", null);
+ reset();
+ });
+
+ const getRequestedUrl = () => {
+ expect(fetchSpy).toHaveBeenCalledTimes(1);
+ return fetchSpy.mock.calls[0][0];
+ };
+
+ it("posts through the instance relay instead of the collector", async () => {
+ Settings.set("metaplow-url", COLLECTOR_URL);
+ renderHook(() =>
+ useInitSdkTracker(SSO_AUTH_CONFIG, makeStore(trackingOn), false),
+ );
+
+ await trackMetaplowEvent("button_clicked");
+
+ expect(getRequestedUrl()).toBe(RELAY_SEND_URL);
+ });
+
+ it("routes through the relay on every instance, Cloud included", async () => {
+ renderHook(() =>
+ useInitSdkTracker(SSO_AUTH_CONFIG, makeStore(trackingOn), false),
+ );
+
+ await trackMetaplowEvent("button_clicked");
+
+ expect(getRequestedUrl()).toBe(RELAY_SEND_URL);
+ });
+
+ it("does not double the separator when the instance URL has a trailing slash", async () => {
+ renderHook(() =>
+ useInitSdkTracker(
+ {
+ ...SSO_AUTH_CONFIG,
+ metabaseInstanceUrl: "https://metabase.example.com/",
+ },
+ makeStore(trackingOn),
+ false,
+ ),
+ );
+
+ await trackMetaplowEvent("button_clicked");
+
+ expect(getRequestedUrl()).toBe(RELAY_SEND_URL);
+ });
+
+ it("does not initialize the tracker when the instance has Metabase Track off", async () => {
+ renderHook(() =>
+ useInitSdkTracker(
+ SSO_AUTH_CONFIG,
+ makeStore({ "metaplow-tracking-enabled": false }),
+ false,
+ ),
+ );
+
+ await trackMetaplowEvent("button_clicked");
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
+});
diff --git a/frontend/src/metabase/app.tsx b/frontend/src/metabase/app.tsx
index fc3b2e1225c..68181f8108d 100644
--- a/frontend/src/metabase/app.tsx
+++ b/frontend/src/metabase/app.tsx
@@ -46,7 +46,7 @@ import { captureClickModifierKeys } from "metabase/urls";
import { setBasename } from "metabase/utils/basename";
import { applyDocumentLocales } from "metabase/utils/boot-locale";
import { captureConsoleErrors } from "metabase/utils/errors";
-import { initMetaplow } from "metabase/utils/metaplow";
+import { getCollectorHostUrl, initMetaplow } from "metabase/utils/metaplow";
import { initTracing, rotateTraceId } from "metabase/utils/otel";
import { PERFORMANCE_MARKS, markOnce } from "metabase/utils/performance-marks";
import MetabaseSettings from "metabase/utils/settings";
@@ -114,6 +114,7 @@ async function _init(
createSnowplowTracker(() => getUserId(store.getState()));
initMetaplow({
+ hostUrl: getCollectorHostUrl(),
getUserId: () => getUserId(store.getState()),
});
diff --git a/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.tsx b/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.tsx
index c9207ce7e03..b3735cb3321 100644
--- a/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.tsx
+++ b/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.tsx
@@ -27,6 +27,7 @@ import { PLUGIN_EMBEDDING_IFRAME_SDK } from "metabase/plugins";
import { useSelector } from "metabase/redux";
import { getSetting } from "metabase/settings";
import { Stack } from "metabase/ui";
+import { getCollectorHostUrl, initMetaplow } from "metabase/utils/metaplow";
import { useParamRerenderKey } from "../hooks/use-param-rerender-key";
import { useSdkIframeEmbedEventBus } from "../hooks/use-sdk-iframe-embed-event-bus";
@@ -78,6 +79,10 @@ const onSettingsChanged = (settings: SdkIframeEmbedSettings) => {
const store = getSdkStore();
createSnowplowTracker(() => getUserId(store.getState()));
+initMetaplow({
+ hostUrl: getCollectorHostUrl(),
+ getUserId: () => getUserId(store.getState()),
+});
export const SdkIframeEmbedRoute = () => {
const { embedSettings } = useSdkIframeEmbedEventBus({
diff --git a/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.unit.spec.tsx b/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.unit.spec.tsx
new file mode 100644
index 00000000000..41ecf177755
--- /dev/null
+++ b/frontend/src/metabase/embedding/embedding-iframe-sdk/components/SdkIframeEmbedRoute.unit.spec.tsx
@@ -0,0 +1,22 @@
+import { getCollectorHostUrl, initMetaplow } from "metabase/utils/metaplow";
+
+jest.mock("metabase/utils/metaplow", () => ({
+ getCollectorHostUrl: jest.fn(() => "https://collector.example.com/api"),
+ initMetaplow: jest.fn(),
+ trackMetaplowEvent: jest.fn(),
+ trackMetaplowPageView: jest.fn(),
+}));
+
+describe("SdkIframeEmbedRoute analytics", () => {
+ it("initializes Metabase Track for the iframe leg, posting to the collector directly", async () => {
+ await import("./SdkIframeEmbedRoute");
+
+ expect(getCollectorHostUrl).toHaveBeenCalled();
+ expect(initMetaplow).toHaveBeenCalledWith(
+ expect.objectContaining({
+ hostUrl: "https://collector.example.com/api",
+ getUserId: expect.any(Function),
+ }),
+ );
+ });
+});
diff --git a/frontend/src/metabase/utils/metaplow.ts b/frontend/src/metabase/utils/metaplow.ts
index 5bb41e4a043..7665c358b11 100644
--- a/frontend/src/metabase/utils/metaplow.ts
+++ b/frontend/src/metabase/utils/metaplow.ts
@@ -17,15 +17,22 @@ const getSanitizedUrl = (url: string) => {
};
interface MetaplowConfig {
+ hostUrl: string | null;
getUserId: () => number | undefined;
}
-export function initMetaplow(config: MetaplowConfig): void {
+export function getCollectorHostUrl(): string | null {
+ if (!Settings.get("metaplow-tracking-enabled")) {
+ return null;
+ }
const metaplowUrl = Settings.get("metaplow-url");
- if (!metaplowUrl) {
+ return metaplowUrl ? metaplowUrl.replace(/\/send$/, "") : null;
+}
+
+export function initMetaplow({ hostUrl, getUserId }: MetaplowConfig): void {
+ if (!hostUrl) {
return;
}
- const hostUrl = metaplowUrl.replace(/\/send$/, "");
init({
website: METAPLOW_WEBSITE_ID,
@@ -59,7 +66,7 @@ export function initMetaplow(config: MetaplowConfig): void {
data: {
// Unjustified type cast. FIXME
...(payload.data as Record<string, unknown> | undefined),
- user_id: config.getUserId(),
+ user_id: getUserId(),
},
};
},
diff --git a/frontend/src/metabase/utils/metaplow.unit.spec.ts b/frontend/src/metabase/utils/metaplow.unit.spec.ts
index 3a9d4b8a03f..46df25fc310 100644
--- a/frontend/src/metabase/utils/metaplow.unit.spec.ts
+++ b/frontend/src/metabase/utils/metaplow.unit.spec.ts
@@ -3,6 +3,7 @@ import { reset } from "@metabase/track";
import Settings from "metabase/utils/settings";
import {
+ getCollectorHostUrl,
initMetaplow,
trackMetaplowEvent,
trackMetaplowPageView,
@@ -11,9 +12,13 @@ import {
const METAPLOW_URL = "https://metaplow.example.com/api/send";
const METAPLOW_WEBSITE_ID = "23eefa30-4c4f-490e-aa4f-084cd23b1561";
const ANON_ORIGIN = "http://anonymous.metabase.com";
+const RELAY_HOST_URL = "https://app.example.com/api/metaplow-proxy";
-const setup = async ({ userId }: { userId?: number } = {}) => {
- initMetaplow({ getUserId: () => userId });
+const setup = async ({
+ userId,
+ hostUrl = getCollectorHostUrl(),
+}: { userId?: number; hostUrl?: string | null } = {}) => {
+ initMetaplow({ hostUrl, getUserId: () => userId });
await Promise.resolve();
};
@@ -21,6 +26,7 @@ describe("metaplow", () => {
let fetchSpy: jest.SpyInstance;
beforeEach(() => {
+ Settings.set("metaplow-tracking-enabled", true);
Settings.set("metaplow-url", METAPLOW_URL);
fetchSpy = jest
.spyOn(global, "fetch")
@@ -29,6 +35,7 @@ describe("metaplow", () => {
afterEach(() => {
fetchSpy.mockRestore();
+ Settings.set("metaplow-tracking-enabled", false);
Settings.set("metaplow-url", null);
reset();
});
@@ -46,10 +53,26 @@ describe("metaplow", () => {
return JSON.parse(init?.body as string);
};
+ describe("getCollectorHostUrl", () => {
+ it("strips the /send suffix the tracker library adds back on", () => {
+ expect(getCollectorHostUrl()).toBe("https://metaplow.example.com/api");
+ });
+
+ it("is null when no collector is configured", () => {
+ Settings.set("metaplow-url", null);
+ expect(getCollectorHostUrl()).toBeNull();
+ });
+
+ it("is null when the instance has Metabase Track off, even with a collector configured", () => {
+ Settings.set("metaplow-tracking-enabled", false);
+ expect(getCollectorHostUrl()).toBeNull();
+ });
+ });
+
describe("initMetaplow", () => {
describe("getUserId", () => {
it("injects the user_id into the event data", async () => {
- initMetaplow({ getUserId: () => 123 });
+ await setup({ userId: 123 });
await trackMetaplowEvent("button_clicked");
@@ -57,6 +80,23 @@ describe("metaplow", () => {
expect(payload.data.user_id).toBe(123);
});
});
+
+ it("posts to the host URL it is given rather than to the configured collector", async () => {
+ await setup({ hostUrl: RELAY_HOST_URL });
+
+ await trackMetaplowEvent("button_clicked");
+
+ expect(fetchSpy).toHaveBeenCalledTimes(1);
+ expect(fetchSpy.mock.calls[0][0]).toBe(`${RELAY_HOST_URL}/send`);
+ });
+
+ it("does not initialize without a host URL", async () => {
+ await setup({ hostUrl: null });
+
+ await trackMetaplowEvent("button_clicked");
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
});
describe("trackMetaplowEvent", () => {
diff --git a/metaplow/docker-compose.yml b/metaplow/docker-compose.yml
new file mode 100644
index 00000000000..295ed6c022a
--- /dev/null
+++ b/metaplow/docker-compose.yml
@@ -0,0 +1,11 @@
+services:
+ metaplow-sink:
+ ## Recording stand-in for the Metabase Track collector; see sink.js. Zero dependencies, so a
+ ## stock node image running the mounted script is the whole service.
+ ## Pinned, like snowplow-micro next door: CI should not pick up a new runtime on its own.
+ image: node:22.23.2-alpine
+ ports:
+ - "9091:9091"
+ volumes:
+ - .:/sink:ro
+ command: node /sink/sink.js
diff --git a/metaplow/sink.js b/metaplow/sink.js
new file mode 100644
index 00000000000..11079276918
--- /dev/null
+++ b/metaplow/sink.js
@@ -0,0 +1,148 @@
+/**
+ * Recording sink for Metabase Track (Metaplow) events, used by the end-to-end suite.
+ *
+ * It stands in for the collector: it accepts the posts the browser tracker and the backend sender
+ * make, keeps them in memory, and hands them back so a spec can assert an event actually arrived.
+ * Deliberately not a collector — a real one is an application with a database and migrations, which
+ * is disproportionate CI surface for asserting that a JSON body arrived. The equivalent for Snowplow
+ * is the `snowplow-micro` container in `snowplow/docker-compose.yml`.
+ *
+ * Routes:
+ * POST <anything> record the body and answer `{}` — a JSON object with no `disabled` flag, which
+ * is what the tracker library needs to keep sending. Any path is recorded, not
+ * just the collector's, so a misconfigured collector URL shows up as a wrong
+ * `path` rather than as silence.
+ * GET /sink/events everything recorded since the last reset, oldest first
+ * GET /sink/reset drop everything recorded
+ *
+ * No dependencies: the container runs this file straight off a stock `node` image.
+ */
+
+const http = require("node:http");
+
+/** Plenty for any one spec, and bounds memory across a whole e2e run that never resets. */
+const DEFAULT_MAX_EVENTS = 1000;
+
+/**
+ * A real event is a few KB — the tracker truncates strings at 500 characters — and the largest thing
+ * the backend sends is the daily instance-statistics payload. 1MB is well clear of both.
+ */
+const DEFAULT_MAX_BODY_BYTES = 1024 * 1024;
+
+const CORS_HEADERS = {
+ // The app posts from a different origin (the instance) than the sink's own.
+ "Access-Control-Allow-Origin": "*",
+ "Access-Control-Allow-Methods": "GET, POST, OPTIONS",
+ // `x-track-cache` is the tracker library's own header; without it the browser blocks the post.
+ "Access-Control-Allow-Headers": "content-type, x-track-cache",
+ "Access-Control-Max-Age": "86400",
+};
+
+function respond(res, status, body) {
+ const payload = JSON.stringify(body ?? {});
+ res.writeHead(status, {
+ ...CORS_HEADERS,
+ "Content-Type": "application/json",
+ "Content-Length": Buffer.byteLength(payload),
+ });
+ res.end(payload);
+}
+
+/** Reads the whole body, or resolves `null` once it goes over `maxBodyBytes`. */
+function readBody(req, maxBodyBytes) {
+ return new Promise((resolve, reject) => {
+ const chunks = [];
+ let size = 0;
+ let overflowed = false;
+
+ req.on("data", (chunk) => {
+ size += chunk.length;
+ if (size > maxBodyBytes) {
+ overflowed = true;
+ // Keep draining rather than destroying the socket: the client gets a 413 instead of a
+ // connection reset, which is far easier to read in a failing spec.
+ return;
+ }
+ chunks.push(chunk);
+ });
+ req.on("end", () =>
+ resolve(overflowed ? null : Buffer.concat(chunks).toString("utf8")),
+ );
+ req.on("error", reject);
+ });
+}
+
+function createSink({
+ maxEvents = DEFAULT_MAX_EVENTS,
+ maxBodyBytes = DEFAULT_MAX_BODY_BYTES,
+} = {}) {
+ let events = [];
+
+ const record = (path, raw) => {
+ let body = null;
+ try {
+ body = JSON.parse(raw);
+ } catch {
+ // Left as null, with the text kept in `raw`. A body the collector could not have read is
+ // itself worth seeing in a failing spec.
+ }
+ events.push({
+ receivedAt: new Date().toISOString(),
+ path,
+ body,
+ ...(body === null ? { raw } : {}),
+ });
+ if (events.length > maxEvents) {
+ events = events.slice(events.length - maxEvents);
+ }
+ };
+
+ return http.createServer((req, res) => {
+ const path = (req.url ?? "/").split("?")[0];
+
+ if (req.method === "OPTIONS") {
+ res.writeHead(204, CORS_HEADERS);
+ res.end();
+ return;
+ }
+
+ if (req.method === "GET" && path === "/sink/events") {
+ respond(res, 200, events);
+ return;
+ }
+
+ if (req.method === "GET" && path === "/sink/reset") {
+ events = [];
+ respond(res, 200, { reset: true });
+ return;
+ }
+
+ if (req.method === "POST") {
+ readBody(req, maxBodyBytes)
+ .then((raw) => {
+ if (raw === null) {
+ respond(res, 413, { error: "body too large" });
+ return;
+ }
+ record(path, raw);
+ // `{}` rather than an empty body: the tracker parses the response and reads `disabled` and
+ // `cache` out of it, so it has to be JSON, and it must not say `disabled`.
+ respond(res, 200, {});
+ })
+ .catch(() => respond(res, 400, { error: "could not read body" }));
+ return;
+ }
+
+ respond(res, 404, { error: "not found" });
+ });
+}
+
+module.exports = { createSink };
+
+if (require.main === module) {
+ const port = Number(process.env.PORT ?? 9091);
+ createSink().listen(port, () => {
+ // eslint-disable-next-line metabase/no-literal-metabase-strings -- a dev container's startup log, never shown to a user
+ console.log(`Metabase Track sink listening on ${port}`);
+ });
+}
diff --git a/metaplow/sink.unit.spec.js b/metaplow/sink.unit.spec.js
new file mode 100644
index 00000000000..55ba2168586
--- /dev/null
+++ b/metaplow/sink.unit.spec.js
@@ -0,0 +1,190 @@
+const http = require("node:http");
+
+const { createSink } = require("./sink");
+
+const listen = (sink) =>
+ new Promise((resolve) => {
+ sink.listen(0, "127.0.0.1", () => resolve(sink.address().port));
+ });
+
+const closeSink = (sink) => new Promise((resolve) => sink.close(resolve));
+
+/** Minimal `node:http` client so these tests do not depend on a global `fetch`. */
+const request = (port, { method = "GET", path = "/", body, headers = {} }) =>
+ new Promise((resolve, reject) => {
+ const req = http.request(
+ { host: "127.0.0.1", port, method, path, headers },
+ (res) => {
+ let text = "";
+ res.setEncoding("utf8");
+ res.on("data", (chunk) => (text += chunk));
+ res.on("end", () =>
+ resolve({ status: res.statusCode, headers: res.headers, text }),
+ );
+ },
+ );
+ req.on("error", reject);
+ if (body !== undefined) {
+ req.write(body);
+ }
+ req.end();
+ });
+
+const post = (port, payload, path = "/api/send") =>
+ request(port, {
+ method: "POST",
+ path,
+ body: typeof payload === "string" ? payload : JSON.stringify(payload),
+ headers: { "content-type": "application/json" },
+ });
+
+const recorded = async (port) => {
+ const { status, text } = await request(port, { path: "/sink/events" });
+ expect(status).toBe(200);
+ return JSON.parse(text);
+};
+
+const eventPayload = (name) => ({ type: "event", payload: { name } });
+
+describe("metaplow sink", () => {
+ let sink;
+ let port;
+
+ const start = async (options) => {
+ sink = createSink(options);
+ port = await listen(sink);
+ return port;
+ };
+
+ afterEach(async () => {
+ if (sink) {
+ await closeSink(sink);
+ sink = undefined;
+ }
+ });
+
+ it("records a posted event and exposes it", async () => {
+ await start();
+
+ await post(port, eventPayload("dashboard.dashboard_created"));
+
+ const events = await recorded(port);
+ expect(events).toHaveLength(1);
+ expect(events[0]).toMatchObject({
+ path: "/api/send",
+ body: { type: "event", payload: { name: "dashboard.dashboard_created" } },
+ });
+ expect(typeof events[0].receivedAt).toBe("string");
+ });
+
+ it("answers a post with a JSON body the tracker can parse without disabling itself", async () => {
+ await start();
+
+ const { status, headers, text } = await post(
+ port,
+ eventPayload("pageview"),
+ );
+
+ expect(status).toBe(200);
+ expect(headers["content-type"]).toMatch(/application\/json/);
+ expect(JSON.parse(text).disabled).toBeFalsy();
+ });
+
+ it("keeps posts in arrival order", async () => {
+ await start();
+
+ await post(port, eventPayload("first"));
+ await post(port, eventPayload("second"));
+
+ const events = await recorded(port);
+ expect(events.map((e) => e.body.payload.name)).toEqual(["first", "second"]);
+ });
+
+ it("clears what it recorded on reset", async () => {
+ await start();
+ await post(port, eventPayload("pageview"));
+
+ const { status } = await request(port, { path: "/sink/reset" });
+
+ expect(status).toBe(200);
+ expect(await recorded(port)).toEqual([]);
+ });
+
+ it("does not record its own control requests", async () => {
+ await start();
+
+ await request(port, { path: "/sink/reset" });
+ await request(port, { path: "/sink/events" });
+
+ expect(await recorded(port)).toEqual([]);
+ });
+
+ it("records a post to any path, so a misconfigured collector URL is still visible", async () => {
+ await start();
+
+ await post(port, eventPayload("pageview"), "/somewhere/else");
+
+ expect(await recorded(port)).toMatchObject([{ path: "/somewhere/else" }]);
+ });
+
+ it("records a body that is not JSON as raw text rather than failing", async () => {
+ await start();
+
+ const { status } = await post(port, "not json at all");
+
+ expect(status).toBe(200);
+ expect(await recorded(port)).toMatchObject([
+ { body: null, raw: "not json at all" },
+ ]);
+ });
+
+ it("rejects and drops a body over the size cap", async () => {
+ await start({ maxBodyBytes: 32 });
+
+ const { status } = await post(port, eventPayload("x".repeat(100)));
+
+ expect(status).toBe(413);
+ expect(await recorded(port)).toEqual([]);
+ });
+
+ it("retains only the most recent events once the cap is reached", async () => {
+ await start({ maxEvents: 2 });
+
+ await post(port, eventPayload("first"));
+ await post(port, eventPayload("second"));
+ await post(port, eventPayload("third"));
+
+ const events = await recorded(port);
+ expect(events.map((e) => e.body.payload.name)).toEqual(["second", "third"]);
+ });
+
+ it("allows cross-origin posts from the app, including the tracker's cache header", async () => {
+ await start();
+
+ const preflight = await request(port, {
+ method: "OPTIONS",
+ path: "/api/send",
+ headers: {
+ origin: "http://localhost:4000",
+ "access-control-request-method": "POST",
+ "access-control-request-headers": "content-type,x-track-cache",
+ },
+ });
+
+ expect(preflight.status).toBe(204);
+ expect(preflight.headers["access-control-allow-origin"]).toBe("*");
+ expect(preflight.headers["access-control-allow-headers"]).toMatch(
+ /x-track-cache/i,
+ );
+ const { headers } = await post(port, eventPayload("pageview"));
+ expect(headers["access-control-allow-origin"]).toBe("*");
+ });
+
+ it("answers an unknown route with a 404", async () => {
+ await start();
+
+ const { status } = await request(port, { path: "/nope" });
+
+ expect(status).toBe(404);
+ });
+});
diff --git a/src/metabase/analytics/api/metaplow_proxy.clj b/src/metabase/analytics/api/metaplow_proxy.clj
new file mode 100644
index 00000000000..a7a84d49135
--- /dev/null
+++ b/src/metabase/analytics/api/metaplow_proxy.clj
@@ -0,0 +1,191 @@
+(ns metabase.analytics.api.metaplow-proxy
+ "Public, anonymous Metabase Track telemetry relay for the Embedding SDK.
+
+ The SDK runs inside the customer's page, where the customer's `connect-src` CSP blocks a direct browser POST to the
+ Metabase Track collector. The SDK instead points its tracker at this instance — the same host its data calls
+ already go to, so it is already CSP-allowlisted — and we forward the payload to the collector server-side, where no
+ browser CSP applies. Same arrangement as [[metabase.analytics.api.proxy]], which does this for Snowplow.
+
+ The route is `/send` rather than `/` because the tracker library composes its endpoint as its configured host URL
+ plus `/send`; the SDK configures `<instance>/api/metaplow-proxy` as that host URL.
+
+ Status and body are relayed verbatim: the tracker reads a `disabled` flag and a `cache` token out of the
+ collector's response body, so a synthesized response would break that protocol.
+
+ Unlike the Snowplow `tp2` payload, Metabase Track event data is arbitrary nested JSON by design, so there is no
+ closed schema to validate it against. We validate the envelope the tracker always wraps it in and bound the rest
+ with an explicit size cap.
+
+ Body handling: `wrap-json-body` parses the body before this handler runs, so `body` arrives as a map and we
+ re-encode it — not a byte-exact relay. Safe here because the tracker produces the body with `JSON.stringify` and
+ the collector reads it as JSON, so a parse→encode round trip is semantically lossless."
+ (:require
+ [clj-http.client :as http]
+ [clj-http.conn-mgr :as conn-mgr]
+ [diehard.bulkhead :as dh.bh]
+ [diehard.core :as dh]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.analytics.settings :as analytics.settings]
+ [metabase.api.macros :as api.macros]
+ [metabase.request.core :as request]
+ [metabase.util :as u]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]
+ [metabase.util.throttle :as u.throttle]
+ [throttle.core :as throttle])
+ (:import
+ (clojure.lang ExceptionInfo)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private forward-timeout-ms 5000)
+
+;; This endpoint is unauthenticated and makes its outbound call inline on the request thread, so an anonymous caller
+;; can hold a request thread per call for as long as the forward takes. Same settings as the throttler
+;; on [[metabase.analytics.api.proxy]], the sibling relay, so the two public relays do not differ. This bounds the
+;; request *rate* per key, not the number of requests in flight; [[forward-bulkhead]] bounds that.
+(def ^:private throttler
+ (throttle/make-throttler :metaplow-proxy
+ :attempts-threshold 600
+ :attempt-ttl-ms (* 60 1000)
+ ;; don't punish when throttling, just shed load
+ :initial-delay-ms 1000
+ :delay-exponent 1))
+
+(def ^:private max-concurrent-forwards
+ "Total concurrent forwards allowed in flight at once."
+ 8)
+
+(defonce ^:private
+ ^{:doc "Process-wide bulkhead bounding concurrent forwards; see [[max-concurrent-forwards]]."}
+ forward-bulkhead
+ (dh.bh/bulkhead {:concurrency max-concurrent-forwards}))
+
+(defonce ^:private
+ ^{:doc "Reuses TCP/TLS connections to the collector across forwards. Sized to [[max-concurrent-forwards]]: the
+ bulkhead already caps forwards in flight, and every request goes to one host, so `:default-per-route` is the
+ actual cap. Wrapped in `delay` so the pool isn't created until the first forward."}
+ connection-manager
+ (delay (conn-mgr/make-reusable-conn-manager {:threads max-concurrent-forwards
+ :default-per-route max-concurrent-forwards})))
+
+(defn- +throttle
+ "Middleware that answers 429 for a source address over the rate limit, before request validation and the forward."
+ [handler]
+ (fn [request respond raise]
+ (try
+ ;; The bucket key is only as good as the deployment: `ip-address` reads the source-address header
+ ;; (`X-Forwarded-For` by default) unless `MB_NOT_BEHIND_PROXY` is set, so it is the real client behind a proxy
+ ;; that overwrites that header, and something a caller can pick for itself on a directly-exposed instance. It
+ ;; is still the key every other unauthenticated endpoint here throttles on, and the only one available.
+ (throttle/check throttler (request/ip-address request))
+ (handler request respond raise)
+ (catch ExceptionInfo e
+ (if (u.throttle/throttle-exception? e)
+ ;; No body, unlike the sibling relay's `{:error ...}`. The tracker assigns `cache` from any response it can
+ ;; parse as JSON, so a JSON error body would clear the collector's cache token on every throttled send.
+ ;; Every other synthesized status in this ns sends no body for the same reason; `Retry-After` carries the
+ ;; detail a caller needs.
+ (do
+ (analytics/inc! :metabase-analytics-relay/rejected {:relay :metaplow, :reason :throttled})
+ (respond (u.throttle/throttle-response e nil)))
+ (raise e))))))
+
+(def ^:private max-body-bytes
+ "Largest payload we will relay. The tracker truncates strings at 500 characters and objects at 50 properties, so a
+ real event is a few KB; 64KB leaves generous headroom while still bounding what an anonymous caller can push
+ through the instance."
+ (* 64 1024))
+
+(def ^:private cache-token-re
+ "Shape a cache token has to have before we pass it upstream. The collector issues the token, but it reaches us by
+ way of the customer's page, so treat it as caller-controlled and keep it to a bounded run of token characters."
+ #"[A-Za-z0-9._~+/=-]{1,512}")
+
+(def ^:private TrackEnvelope
+ "The `{:type :payload}` wrapper the tracker library puts around every send. `identify` is not accepted: nothing in
+ Metabase calls it, and an anonymous relay has no reason to carry it. Otherwise deliberately open — `:payload`
+ carries browser-derived fields the library may add to over time, and `:data` inside it is arbitrary nested JSON."
+ ;; `{:closed false}` is what opts a map out of `defendpoint`'s strip-extra-keys decoding — without it the event
+ ;; data would be stripped off before we ever forwarded it.
+ [:map {:closed false, ::mr/deliberately-open true, :description "a Metabase Track send envelope"}
+ [:type [:enum "event"]]
+ [:payload [:map {:closed false, ::mr/deliberately-open true, :description "a Metabase Track event payload"}
+ [:website ms/NonBlankString]]]])
+
+(defn- cache-token
+ "The tracker echoes the collector's cache token back in `x-track-cache` on later sends, letting the collector skip a
+ session lookup. Forward it when it looks like a token, drop it otherwise."
+ []
+ (when-let [token (some-> (request/current-request) :headers (get "x-track-cache"))]
+ (when (re-matches cache-token-re token)
+ token)))
+
+(defn- forward!
+ [collector-url ^String encoded-body token]
+ (try
+ (let [response (http/post collector-url
+ (cond-> {:body encoded-body
+ :content-type :json
+ :connection-timeout forward-timeout-ms
+ :socket-timeout forward-timeout-ms
+ ;; Each redirect hop would get its own full timeout, and the hop target is
+ ;; chosen by the collector rather than by us. One request, one destination.
+ :redirect-strategy :none
+ :throw-exceptions false
+ :connection-manager @connection-manager}
+ token (assoc :headers {"x-track-cache" token})))]
+ {:status (:status response)
+ :body (:body response)})
+ (catch Exception e
+ ;; Debug, not warn: the call rate here is set by anonymous callers, so a collector that is down would otherwise
+ ;; flood the log. The error counter is the signal, with a stage label that separates relayed SDK events from
+ ;; the instance's own sends in [[metabase.analytics.metaplow]].
+ (analytics/inc! :metabase-metaplow/errors {:stage :relay-forward!})
+ (log/debugf "metaplow-proxy failed forwarding to %s: %s" collector-url (ex-message e))
+ ;; The tracker ignores the response on failure, but a fake 2xx would still be a lie about delivery.
+ {:status 502 :body nil})))
+
+(defn- do-with-forward-bulkhead
+ "Run `thunk` holding a [[forward-bulkhead]] permit, answering 503 when the relay already has
+ [[max-concurrent-forwards]] forwards in flight."
+ [thunk]
+ (try
+ (dh/with-bulkhead {:bulkhead forward-bulkhead, :max-wait-ms 0}
+ (thunk))
+ (catch ExceptionInfo e
+ (if (:bulkhead (ex-data e))
+ (do
+ (analytics/inc! :metabase-analytics-relay/rejected {:relay :metaplow, :reason :shed})
+ (log/debug "metaplow-proxy shed a forward: too many already in flight")
+ {:status 503 :body nil})
+ (throw e)))))
+
+;; No response schema: the body is an opaque relay of the collector's own response. The tracker reads its `disabled`
+;; flag and `cache` token straight out of it, so a fabricated schema would be misleading.
+;;
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :post "/send"
+ "Forward a Metabase Track payload to the configured collector server-side, bypassing the customer page's
+ `connect-src` CSP."
+ [_route-params
+ _query-params
+ body :- TrackEnvelope]
+ (if-not (analytics.settings/metaplow-tracking-enabled)
+ ;; Gated on the same switch as the instance's own emission — anonymous tracking on and a collector configured.
+ ;; An admin who opted out must not have the instance relaying a caller's events either.
+ {:status 503 :body nil}
+ (let [encoded (json/encode body)]
+ (if (> (alength (u/string-to-bytes encoded)) max-body-bytes)
+ {:status 413 :body nil}
+ ;; `cache-token` reads the current request, so it has to be called on this thread, not inside the forward.
+ (let [token (cache-token)]
+ (do-with-forward-bulkhead
+ (fn []
+ (forward! (analytics.settings/metaplow-url) encoded token))))))))
+
+(def ^{:arglists '([request respond raise])} routes
+ "Routes for the public Metabase Track telemetry relay."
+ (api.macros/ns-handler *ns* +throttle))
diff --git a/src/metabase/analytics/api/proxy.clj b/src/metabase/analytics/api/proxy.clj
index 8133015063f..ac02874eb02 100644
--- a/src/metabase/analytics/api/proxy.clj
+++ b/src/metabase/analytics/api/proxy.clj
@@ -10,14 +10,68 @@
(base64 blobs survive a parse→encode round-trip), and v1 pins `eventMethod: \"post\"` with a JSON content-type."
(:require
[clj-http.client :as http]
+ [clj-http.conn-mgr :as conn-mgr]
+ [diehard.bulkhead :as dh.bh]
+ [diehard.core :as dh]
+ [metabase.analytics-interface.core :as analytics]
[metabase.analytics.settings :as analytics.settings]
[metabase.api.macros :as api.macros]
+ [metabase.request.core :as request]
[metabase.util.json :as json]
[metabase.util.log :as log]
- [metabase.util.malli.schema :as ms]))
+ [metabase.util.malli.schema :as ms]
+ [metabase.util.throttle :as u.throttle]
+ [throttle.core :as throttle])
+ (:import
+ (clojure.lang ExceptionInfo)))
(def ^:private forward-timeout-ms 5000)
+;; This endpoint is unauthenticated and makes its outbound call inline on the request thread, so an anonymous caller
+;; can hold a request thread per call for as long as the forward takes.
+(def ^:private throttler
+ (throttle/make-throttler :analytics-proxy
+ :attempts-threshold 600
+ :attempt-ttl-ms (* 60 1000)
+ ;; don't punish when throttling, just shed load
+ :initial-delay-ms 1000
+ :delay-exponent 1))
+
+(def ^:private max-concurrent-forwards
+ "Forwards allowed in flight at once."
+ 8)
+
+(defonce ^:private
+ ^{:doc "Process-wide bulkhead bounding concurrent forwards; see [[max-concurrent-forwards]]."}
+ forward-bulkhead
+ (dh.bh/bulkhead {:concurrency max-concurrent-forwards}))
+
+(defonce ^:private
+ ^{:doc "Reuses TCP/TLS connections to the collector across forwards. Sized to [[max-concurrent-forwards]]: the
+ bulkhead already caps forwards in flight, and every request goes to one host, so `:default-per-route` is the
+ actual cap. Wrapped in `delay` so the pool isn't created until the first forward."}
+ connection-manager
+ (delay (conn-mgr/make-reusable-conn-manager {:threads max-concurrent-forwards
+ :default-per-route max-concurrent-forwards})))
+
+(defn- +throttle
+ "Middleware that answers 429 for a source address over the rate limit, before request validation and the forward."
+ [handler]
+ (fn [request respond raise]
+ (try
+ ;; The bucket key is only as good as the deployment: `ip-address` reads the source-address header
+ ;; (`X-Forwarded-For` by default) unless `MB_NOT_BEHIND_PROXY` is set, so it is the real client behind a proxy
+ ;; that overwrites that header, and something a caller can pick for itself on a directly-exposed instance. It
+ ;; is still the key every other unauthenticated endpoint here throttles on, and the only one available.
+ (throttle/check throttler (request/ip-address request))
+ (handler request respond raise)
+ (catch ExceptionInfo e
+ (if (u.throttle/throttle-exception? e)
+ (do
+ (analytics/inc! :metabase-analytics-relay/rejected {:relay :snowplow, :reason :throttled})
+ (respond (u.throttle/throttle-response e {:error (ex-message e)})))
+ (raise e))))))
+
(def ^:private Tp2Payload
"Schema for the Snowplow `tp2` request envelope, mirroring the collector's own `payload_data` JSON schema: the iglu
URI of that schema plus one entry per event, keyed by tracker field name (`e`, `p`, `tv`, `ue_px`, ...). The tracker
@@ -26,6 +80,37 @@
[:schema ms/NonBlankString]
[:data [:sequential {:max 100} (ms/string-keyed-map :string)]]])
+(defn- forward!
+ [collector-url body]
+ (try
+ (let [response (http/post collector-url
+ {:body (json/encode body)
+ :content-type :json
+ :connection-timeout forward-timeout-ms
+ :socket-timeout forward-timeout-ms
+ :throw-exceptions false
+ :connection-manager @connection-manager})]
+ {:status (:status response)
+ :body (:body response)})
+ (catch Exception e
+ (log/errorf "analytics-proxy failed forwarding to %s: %s" collector-url (ex-message e))
+ {:status 502 :body nil})))
+
+(defn- do-with-forward-bulkhead
+ "Run `thunk` holding a [[forward-bulkhead]] permit, answering 503 when the relay already has
+ [[max-concurrent-forwards]] forwards in flight."
+ [thunk]
+ (try
+ (dh/with-bulkhead {:bulkhead forward-bulkhead, :max-wait-ms 0}
+ (thunk))
+ (catch ExceptionInfo e
+ (if (:bulkhead (ex-data e))
+ (do
+ (analytics/inc! :metabase-analytics-relay/rejected {:relay :snowplow, :reason :shed})
+ (log/debug "analytics-proxy shed a forward: too many already in flight")
+ {:status 503 :body nil})
+ (throw e)))))
+
;; No response schema: the body is an opaque relay of the collector's own response. The tracker drives its retry
;; outbox off the HTTP status, so we forward status + body verbatim; a fabricated schema would be misleading.
;;
@@ -37,17 +122,10 @@
_query-params
body :- Tp2Payload]
(let [collector-url (str (analytics.settings/snowplow-url) "/com.snowplowanalytics.snowplow/tp2")]
- (try
- (let [response (http/post collector-url
- {:body (json/encode body)
- :content-type :json
- :connection-timeout forward-timeout-ms
- :socket-timeout forward-timeout-ms
- :throw-exceptions false})]
- ;; Relay status + body unchanged: 2xx clears the tracker's event, non-2xx schedules a retry.
- {:status (:status response)
- :body (:body response)})
- (catch Exception e
- ;; 502 is retryable by the tracker; a fake 2xx would silently drop the event.
- (log/errorf "analytics-proxy failed forwarding to %s: %s" collector-url (ex-message e))
- {:status 502 :body nil}))))
+ (do-with-forward-bulkhead
+ (fn []
+ (forward! collector-url body)))))
+
+(def ^{:arglists '([request respond raise])} routes
+ "Routes for the public Snowplow telemetry relay."
+ (api.macros/ns-handler *ns* +throttle))
diff --git a/src/metabase/analytics/core.clj b/src/metabase/analytics/core.clj
index 96abe2d30da..cbeda9e7893 100644
--- a/src/metabase/analytics/core.clj
+++ b/src/metabase/analytics/core.clj
@@ -53,7 +53,8 @@
[metabase.analytics.settings
anon-tracking-enabled
anon-tracking-enabled!
- instance-creation]
+ instance-creation
+ metaplow-url]
[metabase.analytics.event
track-event!]
[metabase.analytics.stats
diff --git a/src/metabase/analytics/metaplow.clj b/src/metabase/analytics/metaplow.clj
index a1b02d07c58..68fd00e0387 100644
--- a/src/metabase/analytics/metaplow.clj
+++ b/src/metabase/analytics/metaplow.clj
@@ -2,9 +2,12 @@
"Sends analytics events to a Metaplow (Umami-compatible) collector.
`track-event!` builds a payload, offers it onto an async send queue, and returns. A `pipeline-blocking` worker pool
- drains the queue and POSTs each event to the collector's `/api/send` endpoint, retrying transient failures via
- [[metabase.util.retry/with-retry]]. The HTTP connection pool is sized to match the worker count so workers never
- contend for slots.
+ drains the queue and POSTs each event to the collector's `/api/send` endpoint. The HTTP connection pool is sized to
+ match the worker count so workers never contend for slots.
+
+ Delivery is bounded per event by [[retry-policy]] and across events by [[breaker]]. Failed sends log at debug only;
+ the `:metabase-metaplow/errors` counter, labelled by `:stage`, is the operational signal. The breaker's state
+ changes log at warn and info, at most about once a minute.
Emission is gated by [[metabase.analytics.settings/metaplow-tracking-enabled]]. The public entry point used by the
rest of the codebase is [[metabase.analytics.event/track-event!]], which fans out to both Snowplow and Metaplow."
@@ -13,6 +16,8 @@
[clj-http.conn-mgr :as conn-mgr]
[clojure.core.async :as a]
[clojure.walk :as walk]
+ [diehard.circuit-breaker :as dh.cb]
+ [diehard.core :as dh]
[metabase.analytics-interface.core :as analytics]
[metabase.analytics.settings :as analytics.settings]
[metabase.analytics.snowplow :as snowplow]
@@ -23,7 +28,9 @@
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[metabase.util.retry :as retry]
- [metabase.version.core :as version]))
+ [metabase.version.core :as version])
+ (:import
+ (dev.failsafe CircuitBreakerOpenException)))
(set! *warn-on-reflection* true)
@@ -103,30 +110,84 @@
:connection-manager @connection-manager})
(catch Throwable e
(analytics/inc! :metabase-metaplow/errors {:stage :send-event!})
- (log/warnf "Connection failure sending Metaplow event: %s" (ex-message e))
+ (log/debugf "Connection failure sending Metaplow event: %s" (ex-message e))
{:status -1})))
+(defn- delivered?
+ "Truthy when the collector accepted the event, i.e. the response is a 2xx."
+ [{:keys [status]}]
+ (and (some? status) (<= 200 status 299)))
+
(defn- retryable-response?
- "Truthy when a `send-event!` response should trigger another attempt: non-2xx and not one of the no-retry
+ "Truthy when a `send-event!` response should trigger another attempt: not accepted, and not one of the no-retry
status codes that Snowplow also gives up on (400/401/403/410/422)."
- [{:keys [status]}]
+ [{:keys [status] :as response}]
(and (some? status)
- (not (<= 200 status 299))
+ (not (delivered? response))
(not (contains? no-retry-status-codes status))))
+(defn- failed-attempt?
+ "Truthy when one attempt at [[send-event!]] counts as a failure, for both retry and circuit-breaking purposes."
+ [response ex]
+ ;; a no-retry status is not a failure here: the collector answered, so it is reachable, it just won't take this event
+ (or (some? ex) (retryable-response? response)))
+
+(def ^:private retry-policy
+ "Bounds the delivery of one event: at most three attempts, 500ms to 2s apart, with no further attempt started more
+ than 5s in.
+
+ That bounds the retrying, not the wall clock. `:max-duration-ms` is only tested between attempts, so an event whose
+ first attempt stalls out on [[send-event!]]'s connect and socket timeouts gets one attempt rather than three, and
+ can hold a worker for around twice those timeouts."
+ {:max-retries 2
+ :backoff-ms [500 2000]
+ :max-duration-ms 5000
+ :retry-if failed-attempt?})
+
+(def ^:private breaker-config
+ "Opens after ten consecutive failed events, rejects every event for the next 60 seconds, then admits one probe and
+ closes again if it succeeds. Counted in failed events, not failed attempts — see [[breaker]]."
+ {:failure-threshold 10
+ :delay-ms (u/seconds->ms 60)
+ :success-threshold 1
+ :fail-if failed-attempt?
+ :on-open (fn [_] (log/warn "Metaplow collector unreachable; opening circuit breaker"))
+ :on-half-open (fn [_] (log/info "Metaplow circuit breaker half-open; probing the collector"))
+ :on-close (fn [_] (log/info "Metaplow collector reachable again; closing circuit breaker"))})
+
+(defn- new-breaker
+ "A circuit breaker built from [[breaker-config]], with `overrides` merged over it."
+ [overrides]
+ (dh.cb/circuit-breaker (merge breaker-config overrides)))
+
+(defonce ^:private
+ ^{:doc "Process-wide circuit breaker guarding the collector, shared by every worker, so that once it is open they
+ all reject immediately."}
+ breaker
+ (new-breaker nil))
+
(defn- send-event-with-retries!
- "Send an event, retrying with exponential backoff on connection failures or retryable HTTP statuses (via
- [[metabase.util.retry/with-retry]]). Returns `:sent` on success and `:error` on failure (never nil), so callers can
- reliably distinguish outcomes."
+ "Send one event under [[retry-policy]], guarded by [[breaker]]. Returns `:sent` when the collector accepted the
+ event, `:error` when it did not, and `:rejected` when the breaker was open and nothing was sent — never nil, so
+ callers can tell the three apart."
[payload]
(try
- (retry/with-retry {:retry-if (fn [response _ex] (retryable-response? response))}
- (send-event! payload))
- :sent
- (catch Throwable e
- (analytics/inc! :metabase-metaplow/errors {:stage :send-event-with-retries!})
- (log/warnf "Error sending Metaplow event: %s" (ex-message e))
- :error)))
+ ;; the breaker wraps the retry rather than being passed to it as Diehard's `:circuit-breaker` option, which would
+ ;; compose it inside the retry and count one failure per failed attempt instead of one per failed event
+ (let [response (dh/with-circuit-breaker breaker
+ (retry/with-retry retry-policy
+ (send-event! payload)))]
+ (if (delivered? response)
+ :sent
+ (do (analytics/inc! :metabase-metaplow/errors {:stage :send-event-with-retries!})
+ (log/debugf "Giving up on Metaplow event; last status %s" (:status response))
+ :error)))
+ ;; `send-event!` swallows everything and the retry returns the last response rather than throwing when it gives
+ ;; up on one, so the breaker is the only thing in the body that throws
+ (catch CircuitBreakerOpenException _e
+ (analytics/inc! :metabase-metaplow/errors {:stage :circuit-breaker-open})
+ (log/debug "Metaplow circuit breaker is open; dropping event without attempting a send")
+ :rejected)))
(defonce ^:private
^{:doc "Individual payloads queued by `track-event!` land here. Bounded chan of size `queue-size`; new events are
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 26beaff37b3..d2f3c62291e 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -927,6 +927,10 @@
(prometheus/counter :metabase-metaplow/errors
{:description "Metaplow event pipeline errors by stage."
:labels [:stage]})
+ ;; Not labelled by source address: the relays are anonymous and public, so the caller would pick the label value.
+ (prometheus/counter :metabase-analytics-relay/rejected
+ {:description "Public telemetry relay requests rejected without forwarding, by relay and reason."
+ :labels [:relay :reason]})
(prometheus/gauge :metabase-memoize/cache-size
{:description "Number of entries currently held in a monitored in-memory memoization cache."
:labels [:cache]})])
diff --git a/src/metabase/analytics/settings.clj b/src/metabase/analytics/settings.clj
index 3ef4b9f87dc..7a46d0d9ce0 100644
--- a/src/metabase/analytics/settings.clj
+++ b/src/metabase/analytics/settings.clj
@@ -3,6 +3,7 @@
[java-time.api :as t]
[metabase.analytics.db :as analytics.db]
[metabase.config.core :as config]
+ [metabase.premium-features.core :as premium-features]
[metabase.settings.core :as setting :refer [defsetting]]
[metabase.util.date-2 :as u.date]
[metabase.util.experiment :as experiment]
@@ -45,23 +46,35 @@
:visibility :public
:doc false)
-(defsetting metaplow-tracking-enabled
- (deferred-tru
- (str "Boolean indicating whether analytics events are being sent to Metaplow. "
- "True if anonymous tracking is enabled for this instance, and a Metaplow collector URL is set."))
- :type :boolean
- :getter (fn [] (boolean (and (anon-tracking-enabled)
- (setting/get-value-of-type :string :metaplow-url))))
- :visibility :public
- :doc false)
+(defn- default-metaplow-url
+ "The Metabase Track collector in production run mode, and nil in every other run mode — dev, test and e2e runs must
+ not send real telemetry. Air-gapped instances get no default either; an explicitly set URL still applies."
+ []
+ (when (and config/is-prod?
+ (not (premium-features/airgap-enabled)))
+ "https://product-analytics-ingestion.metabase.com/api/send"))
(defsetting metaplow-url
(deferred-tru "The URL of the Metaplow collector to send analytics events to.")
:encryption :when-encryption-key-set
+ ;; resolved per read rather than as `:default`, which is fixed at load time, before the token is known
+ :getter (fn []
+ (or (setting/get-value-of-type :string :metaplow-url)
+ (default-metaplow-url)))
:visibility :public
:audit :never
:doc false)
+(defsetting metaplow-tracking-enabled
+ (deferred-tru
+ (str "Boolean indicating whether analytics events are being sent to Metaplow. "
+ "True if anonymous tracking is enabled for this instance, and a Metaplow collector URL is set or defaulted."))
+ :type :boolean
+ :getter (fn [] (boolean (and (anon-tracking-enabled)
+ (metaplow-url))))
+ :visibility :public
+ :doc false)
+
(defsetting snowplow-url
(deferred-tru "The URL of the Snowplow collector to send analytics events to.")
:encryption :when-encryption-key-set
diff --git a/src/metabase/analytics/snowplow.clj b/src/metabase/analytics/snowplow.clj
index 45aa989e514..d6e7c724600 100644
--- a/src/metabase/analytics/snowplow.clj
+++ b/src/metabase/analytics/snowplow.clj
@@ -247,19 +247,35 @@
[]
(TrackerConfiguration. "sp" "metabase"))
+(def ^:private http-timeout-ms
+ "Connect, socket, and connection-request timeout for the emitter's HTTP client, in milliseconds."
+ 5000)
+
+(defn- request-config
+ ^RequestConfig []
+ (-> (RequestConfig/custom)
+ ;; Set cookie spec to `STANDARD` to avoid warnings about an invalid cookie
+ ;; header in request response (PR #24579)
+ (.setCookieSpec CookieSpecs/STANDARD)
+ ;; All three default to no timeout, so without these a collector that accepts packets but never
+ ;; responds wedges an emitter thread for the life of the process.
+ (.setConnectTimeout http-timeout-ms)
+ (.setSocketTimeout http-timeout-ms)
+ (.setConnectionRequestTimeout http-timeout-ms)
+ (.build)))
+
+(defn- http-client-adapter
+ "The HTTP transport the Snowplow emitter's threads use, pointed at `url`."
+ ^ApacheHttpClientAdapter [url]
+ (let [client (-> (HttpClients/custom)
+ (.setConnectionManager (PoolingHttpClientConnectionManager.))
+ (.setDefaultRequestConfig (request-config))
+ (.build))]
+ (ApacheHttpClientAdapter. url client)))
+
(defn- network-config
[]
- (let [request-config (-> (RequestConfig/custom)
- ;; Set cookie spec to `STANDARD` to avoid warnings about an invalid cookie
- ;; header in request response (PR #24579)
- (.setCookieSpec CookieSpecs/STANDARD)
- (.build))
- client (-> (HttpClients/custom)
- (.setConnectionManager (PoolingHttpClientConnectionManager.))
- (.setDefaultRequestConfig request-config)
- (.build))
- http-client-adapter (ApacheHttpClientAdapter. (analytics.settings/snowplow-url) client)]
- (NetworkConfiguration. http-client-adapter)))
+ (NetworkConfiguration. (http-client-adapter (analytics.settings/snowplow-url))))
(defn- emitter-config
[]
diff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.clj
index 25d696a383a..9cc03a4f82f 100644
--- a/src/metabase/api_routes/routes.clj
+++ b/src/metabase/api_routes/routes.clj
@@ -7,6 +7,7 @@
[metabase.agent-api.query-guards :as agent-api.query-guards]
[metabase.ai-tracing.api]
[metabase.analytics.api]
+ [metabase.analytics.api.metaplow-proxy]
[metabase.analytics.api.proxy]
[metabase.api-keys.api]
[metabase.api.docs]
@@ -86,6 +87,7 @@
metabase.agent-api.api/keep-me
metabase.ai-tracing.api/keep-me
metabase.analytics.api/keep-me
+ metabase.analytics.api.metaplow-proxy/keep-me
metabase.analytics.api.proxy/keep-me
metabase.api-keys.api/keep-me
metabase.api.util/keep-me
@@ -194,7 +196,7 @@
"/agent" (metabase.agent-api.api/+agent-api-enabled metabase.agent-api.api/routes)
"/alert" (+auth metabase.pulse.api/alert-routes)
"/analytics" (+auth 'metabase.analytics.api)
- "/analytics-proxy" (+public-exceptions 'metabase.analytics.api.proxy)
+ "/analytics-proxy" (+public-exceptions metabase.analytics.api.proxy/routes)
"/api-key" (+auth 'metabase.api-keys.api)
"/automagic-dashboards" (+auth metabase.xrays.api/automagic-dashboards-routes)
"/bookmark" (+auth 'metabase.bookmarks.api)
@@ -248,6 +250,7 @@
"/measure" (+auth 'metabase.measures.api)
"/metabase-mcp" (metabase.mcp.v2.api/+mcp-enabled metabase.mcp.v2.api/handler)
"/metabot" metabase.metabot.api/routes
+ "/metaplow-proxy" (+public-exceptions metabase.analytics.api.metaplow-proxy/routes)
"/metric" (+auth 'metabase.metrics.api)
"/model-index" (+auth 'metabase.indexed-entities.api)
"/native-query-snippet" (+auth 'metabase.native-query-snippets.api)
diff --git a/src/metabase/server/middleware/security.clj b/src/metabase/server/middleware/security.clj
index 6dba27e9990..90030cf9b52 100644
--- a/src/metabase/server/middleware/security.clj
+++ b/src/metabase/server/middleware/security.clj
@@ -342,7 +342,7 @@
(when (analytics/anon-tracking-enabled)
(setting/get-value-of-type :string :snowplow-url))
(when (analytics/anon-tracking-enabled)
- (setting/get-value-of-type :string :metaplow-url))
+ (analytics/metaplow-url))
;; Webpack dev server
(when config/is-dev?
(str "*:" frontend-dev-port " ws://*:" frontend-dev-port))
diff --git a/test/metabase/analytics/api/metaplow_proxy_test.clj b/test/metabase/analytics/api/metaplow_proxy_test.clj
new file mode 100644
index 00000000000..cf3ccdbd049
--- /dev/null
+++ b/test/metabase/analytics/api/metaplow_proxy_test.clj
@@ -0,0 +1,233 @@
+(ns metabase.analytics.api.metaplow-proxy-test
+ "Integration tests for the public Metabase Track telemetry relay (`POST /api/metaplow-proxy/send`).
+
+ Mirrors `metabase.analytics.api.proxy-test`, which does the same for the Snowplow relay: `clj-http.fake` stands in
+ for the collector and `metabase.test.http-client/client` issues anonymous requests."
+ (:require
+ [clj-http.fake :as fake]
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [diehard.bulkhead :as dh.bh]
+ [metabase.analytics.api.metaplow-proxy :as analytics.api.metaplow-proxy]
+ [metabase.test :as mt]
+ [metabase.test.http-client :as client]
+ [metabase.util.json :as json]
+ [throttle.core :as throttle]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private collector-url "http://fake-collector.test/api/send")
+
+(def ^:private sample-payload
+ "A minimal Metabase Track envelope: the `{:type :payload}` wrapper the tracker library always sends."
+ {:type "event"
+ :payload {:website "23eefa30-4c4f-490e-aa4f-084cd23b1561"
+ :name "button_clicked"
+ :url "http://anonymous.metabase.com/dashboard/1"
+ :data {:nested {:arbitrary ["json" 1 true]}}}})
+
+(defn- with-collector!
+ "Pin `metaplow-url` to the fake collector, serve `response` from it, and call `f` with an atom holding the inbound
+ request — still nil if nothing was forwarded."
+ [response f]
+ (let [captured (atom nil)]
+ (mt/with-temporary-setting-values [metaplow-url collector-url]
+ (fake/with-fake-routes
+ {collector-url (fn [request]
+ (reset! captured request)
+ response)}
+ (f captured)))))
+
+(deftest anonymous-public-route-test
+ (testing "an anonymous POST reaches the handler (no auth required) and gets the collector's 2xx back"
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [_captured]
+ ;; client/client with no user = anonymous. A 401 here would mean the route isn't public.
+ (is (= {} (client/client :post 200 "metaplow-proxy/send" sample-payload)))))))
+
+(deftest forwards-payload-to-collector-test
+ (testing "the handler forwards the body to the configured collector as JSON, semantically unchanged"
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [captured]
+ (client/client :post 200 "metaplow-proxy/send" sample-payload)
+ (testing "outbound URL is the configured collector URL"
+ (is (= collector-url
+ (str (name (:scheme @captured)) "://" (:server-name @captured) (:uri @captured)))))
+ (testing "outbound body round-trips to the same JSON the SDK sent (re-encode is semantically lossless)"
+ (is (= (json/decode (json/encode sample-payload))
+ (json/decode (slurp (:body @captured))))))
+ (testing "outbound content-type is JSON"
+ (is (re-find #"(?i)application/json"
+ (get-in @captured [:headers "content-type"] ""))))))))
+
+(deftest relays-status-and-body-verbatim-test
+ (testing "the tracker's disable/cache protocol needs the collector's own response, not a synthesized one"
+ (testing "a 2xx body is relayed unchanged"
+ (with-collector! {:status 200 :headers {} :body "{\"cache\":\"abc123\",\"disabled\":false}"}
+ (fn [_captured]
+ (is (= {:cache "abc123" :disabled false}
+ (client/client :post 200 "metaplow-proxy/send" sample-payload))))))
+ (testing "a non-2xx status and body are relayed unchanged"
+ (with-collector! {:status 400 :headers {} :body "{\"error\":\"bad payload\"}"}
+ (fn [_captured]
+ (is (= {:error "bad payload"}
+ (client/client :post 400 "metaplow-proxy/send" sample-payload))))))))
+
+(deftest forwards-cache-token-test
+ (testing "the tracker echoes the collector's cache token in `x-track-cache`; it is forwarded so the collector can
+ skip a session lookup"
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [captured]
+ (client/client :post 200 "metaplow-proxy/send"
+ {:request-options {:headers {"x-track-cache" "abc123"}}}
+ sample-payload)
+ (is (= "abc123" (get-in @captured [:headers "x-track-cache"]))))))
+ (testing "a cache token that is not a plain token string is dropped rather than forwarded"
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [captured]
+ (client/client :post 200 "metaplow-proxy/send"
+ {:request-options {:headers {"x-track-cache" "not a token"}}}
+ sample-payload)
+ (is (nil? (get-in @captured [:headers "x-track-cache"])))))))
+
+(deftest rejects-invalid-envelope-test
+ (testing "the envelope is validated even though the event data inside it is arbitrary JSON"
+ (doseq [[what payload] {"a missing payload" {:type "event"}
+ "a missing type" {:payload {:website "abc"}}
+ "an unknown type" {:type "pageview" :payload {:website "abc"}}
+ "an identify call" {:type "identify" :payload {:website "abc"}}
+ "a blank website" {:type "event" :payload {:website ""}}
+ "a payload that is not a map" {:type "event" :payload "nope"}}]
+ (testing what
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [captured]
+ (client/client :post 400 "metaplow-proxy/send" payload)
+ (is (nil? @captured) "nothing should reach the collector")))))))
+
+(deftest body-size-cap-test
+ (testing "an oversized body is rejected with 413 and never reaches the collector"
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [captured]
+ ;; well past the 64KB cap
+ (client/client :post 413 "metaplow-proxy/send"
+ (assoc-in sample-payload [:payload :data :blob] (str/join (repeat 100000 "x"))))
+ (is (nil? @captured) "nothing should reach the collector"))))
+ (testing "a payload comfortably under the cap is relayed"
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [_captured]
+ (is (= {} (client/client :post 200 "metaplow-proxy/send"
+ (assoc-in sample-payload [:payload :data :blob] (str/join (repeat 1000 "x"))))))))))
+
+(deftest gated-on-metaplow-tracking-test
+ (testing "with no collector URL configured there is nothing to relay to"
+ (mt/with-temporary-setting-values [metaplow-url nil]
+ (client/client :post 503 "metaplow-proxy/send" sample-payload)))
+ (testing "an instance that opted out of anonymous tracking does not relay a caller's events either"
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [captured]
+ (mt/with-temporary-setting-values [anon-tracking-enabled false]
+ (client/client :post 503 "metaplow-proxy/send" sample-payload))
+ (is (nil? @captured) "nothing should reach the collector")))))
+
+(deftest unreachable-collector-returns-502-test
+ (testing "when the collector is unreachable the relay returns 502 rather than throwing"
+ (mt/with-temporary-setting-values [metaplow-url collector-url]
+ (fake/with-fake-routes
+ {collector-url (fn [_request]
+ (throw (java.net.ConnectException. "connection refused")))}
+ (client/client :post 502 "metaplow-proxy/send" sample-payload)))))
+
+(deftest no-client-ip-forwarded-test
+ (testing "the relay does not add X-Forwarded-For (the payload is already anonymized by the tracker)"
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [captured]
+ (client/client :post 200 "metaplow-proxy/send" sample-payload)
+ (is (nil? (get-in @captured [:headers "x-forwarded-for"])))))))
+
+(def ^:private throttled-response
+ "A throttled response carries `Retry-After` and no body: the tracker assigns its cache token from any response body
+ it can parse, so a body here would clear the collector's token on every throttled send."
+ {:status 429
+ :headers {"Retry-After" string?}
+ :body nil})
+
+(defn- request-options
+ "Request options that make `ip` the source address the server derives for the request. `ip-address` reads
+ `X-Forwarded-For` by default, which is also why the throttle bucket is only as trustworthy as the deployment."
+ [ip]
+ {:request-options {:headers {"x-forwarded-for" ip}}})
+
+(defmacro ^:private with-one-request-budget
+ "Pin the relay throttler to a single request per source address, so one request exhausts a caller's budget."
+ [& body]
+ `(with-redefs [analytics.api.metaplow-proxy/throttler
+ (throttle/make-throttler :metaplow-proxy :attempts-threshold 1)]
+ ~@body))
+
+(defmacro ^:private with-forward-capacity
+ "Pin the relay's bulkhead to `n` concurrent forwards. `0` leaves no permit, so every request is shed."
+ [n & body]
+ `(with-redefs [analytics.api.metaplow-proxy/forward-bulkhead (dh.bh/bulkhead {:concurrency ~n})]
+ ~@body))
+
+(deftest sheds-forward-without-bulkhead-permit-test
+ (testing "a request that cannot take a bulkhead permit is shed with a 503 and never reaches the collector"
+ (with-forward-capacity 0
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [captured]
+ (is (=? {:status 503 :body nil}
+ (client/client-full-response :post 503 "metaplow-proxy/send" sample-payload)))
+ (is (nil? @captured) "shedding must happen before the outbound call, not after it"))))))
+
+(deftest releases-bulkhead-permit-after-forward-test
+ (testing "each forward hands its permit back, so a single-permit bulkhead still serves serial requests"
+ (with-forward-capacity 1
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [_captured]
+ (dotimes [_ 3]
+ (is (= {} (client/client :post 200 "metaplow-proxy/send" sample-payload)))))))))
+
+(deftest throttles-anonymous-caller-test
+ (testing "an anonymous caller is let through under the per-IP rate and rejected with a 429 above it"
+ (with-one-request-budget
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [_captured]
+ (let [opts (request-options "10.1.2.3")]
+ (testing "permitted path"
+ (is (= {} (client/client :post 200 "metaplow-proxy/send" opts sample-payload))))
+ (testing "throttled path"
+ (is (=? throttled-response
+ (client/client-full-response :post 429 "metaplow-proxy/send" opts sample-payload))))))))))
+
+(deftest throttle-is-keyed-per-source-address-test
+ (testing "one caller exhausting its budget does not throttle a different source address"
+ (with-one-request-budget
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [_captured]
+ (is (= {} (client/client :post 200 "metaplow-proxy/send" (request-options "10.1.2.3") sample-payload)))
+ (is (=? throttled-response
+ (client/client-full-response :post 429 "metaplow-proxy/send" (request-options "10.1.2.3")
+ sample-payload)))
+ (is (= {} (client/client :post 200 "metaplow-proxy/send" (request-options "10.9.9.9")
+ sample-payload))))))))
+
+(deftest throttled-request-never-reaches-the-collector-test
+ (testing "a throttled request is rejected before the handler forwards anything, so it holds no outbound connection"
+ (with-one-request-budget
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [captured]
+ (let [opts (request-options "10.1.2.3")]
+ (client/client :post 200 "metaplow-proxy/send" opts sample-payload)
+ (reset! captured nil)
+ (client/client-full-response :post 429 "metaplow-proxy/send" opts sample-payload)
+ (is (nil? @captured) "nothing should reach the collector")))))))
+
+(deftest throttle-runs-before-body-validation-test
+ (testing "a rejected payload still spends the caller's budget, so malformed floods cannot evade the throttle"
+ (with-one-request-budget
+ (with-collector! {:status 200 :headers {} :body "{}"}
+ (fn [_captured]
+ (let [opts (request-options "10.1.2.3")]
+ (client/client :post 400 "metaplow-proxy/send" opts {:type "event"})
+ (is (=? throttled-response
+ (client/client-full-response :post 429 "metaplow-proxy/send" opts sample-payload)))))))))
diff --git a/test/metabase/analytics/api/proxy_test.clj b/test/metabase/analytics/api/proxy_test.clj
index 45142836416..3adcbed0e3b 100644
--- a/test/metabase/analytics/api/proxy_test.clj
+++ b/test/metabase/analytics/api/proxy_test.clj
@@ -6,10 +6,14 @@
requests."
(:require
[clj-http.fake :as fake]
+ [clojure.string :as str]
[clojure.test :refer :all]
+ [diehard.bulkhead :as dh.bh]
+ [metabase.analytics.api.proxy :as analytics.api.proxy]
[metabase.test :as mt]
[metabase.test.http-client :as client]
- [metabase.util.json :as json]))
+ [metabase.util.json :as json]
+ [throttle.core :as throttle]))
(set! *warn-on-reflection* true)
@@ -41,6 +45,29 @@
{(collector-url) (constantly ~response)}
~@body)))))
+(def ^:private throttled-response
+ {:status 429
+ :headers {"Retry-After" string?}
+ :body {:error #(str/starts-with? % "Too many attempts!")}})
+
+(defn- request-options
+ "Request options that make `ip` the server-derived source address for the request."
+ [ip]
+ {:request-options {:headers {"x-forwarded-for" ip}}})
+
+(defmacro ^:private with-one-request-budget
+ "Pin the proxy throttler to a single request per source address, so one request exhausts a caller's budget."
+ [& body]
+ `(with-redefs [analytics.api.proxy/throttler
+ (throttle/make-throttler :analytics-proxy :attempts-threshold 1)]
+ ~@body))
+
+(defmacro ^:private with-forward-capacity
+ "Pin the relay's bulkhead to `n` concurrent forwards. `0` leaves no permit, so every request is shed."
+ [n & body]
+ `(with-redefs [analytics.api.proxy/forward-bulkhead (dh.bh/bulkhead {:concurrency ~n})]
+ ~@body))
+
(deftest anonymous-public-route-test
(testing "an anonymous POST reaches the handler (no auth required) and gets a 2xx back"
(with-collector {:status 200 :headers {} :body "{}"}
@@ -81,3 +108,59 @@
(with-collector captured {:status 200 :headers {} :body "{}"}
(client/client :post 200 "analytics-proxy" sample-payload))
(is (nil? (get-in @captured [:headers "x-forwarded-for"]))))))
+
+(deftest throttles-anonymous-caller-test
+ (testing "an anonymous caller is let through under the per-IP rate and rejected with a 429 above it"
+ (with-one-request-budget
+ (with-collector {:status 200 :headers {} :body "{}"}
+ (let [opts (request-options "10.1.2.3")]
+ (testing "permitted path"
+ (is (= {} (client/client :post 200 "analytics-proxy" opts sample-payload))))
+ (testing "throttled path"
+ (is (=? throttled-response
+ (client/client-full-response :post 429 "analytics-proxy" opts sample-payload)))))))))
+
+(deftest throttle-is-keyed-per-source-address-test
+ (testing "one caller exhausting its budget does not throttle a different source address"
+ (with-one-request-budget
+ (with-collector {:status 200 :headers {} :body "{}"}
+ (is (= {} (client/client :post 200 "analytics-proxy" (request-options "10.1.2.3") sample-payload)))
+ (is (=? throttled-response
+ (client/client-full-response :post 429 "analytics-proxy" (request-options "10.1.2.3")
+ sample-payload)))
+ (is (= {} (client/client :post 200 "analytics-proxy" (request-options "10.9.9.9") sample-payload)))))))
+
+(deftest throttled-request-never-reaches-the-collector-test
+ (testing "a throttled request is rejected before the handler forwards anything, so it holds no outbound connection"
+ (let [captured (atom nil)]
+ (with-collector captured {:status 200 :headers {} :body "{}"}
+ (with-one-request-budget
+ (let [opts (request-options "10.1.2.3")]
+ (client/client :post 200 "analytics-proxy" opts sample-payload)
+ (reset! captured nil)
+ (client/client-full-response :post 429 "analytics-proxy" opts sample-payload)
+ (is (nil? @captured))))))))
+
+(deftest throttle-runs-before-body-validation-test
+ (testing "a rejected payload still spends the caller's budget, so malformed floods cannot evade the throttle"
+ (with-one-request-budget
+ (with-collector {:status 200 :headers {} :body "{}"}
+ (let [opts (request-options "10.1.2.3")]
+ (client/client :post 400 "analytics-proxy" opts {:schema "" :data []})
+ (is (=? throttled-response
+ (client/client-full-response :post 429 "analytics-proxy" opts sample-payload))))))))
+(deftest sheds-forward-without-bulkhead-permit-test
+ (testing "a request that cannot take a bulkhead permit is shed with a 503 and never reaches the collector"
+ (let [captured (atom nil)]
+ (with-forward-capacity 0
+ (with-collector captured {:status 200 :headers {} :body "{}"}
+ (is (=? {:status 503}
+ (client/client-full-response :post 503 "analytics-proxy" sample-payload)))))
+ (is (nil? @captured) "shedding must happen before the outbound call, not after it"))))
+
+(deftest releases-bulkhead-permit-after-forward-test
+ (testing "each forward hands its permit back, so a single-permit bulkhead still serves serial requests"
+ (with-forward-capacity 1
+ (with-collector {:status 200 :headers {} :body "{}"}
+ (dotimes [_ 3]
+ (is (= {} (client/client :post 200 "analytics-proxy" sample-payload))))))))
diff --git a/test/metabase/analytics/metaplow_test.clj b/test/metabase/analytics/metaplow_test.clj
index 66f12b2cf98..f0e2c7c6d43 100644
--- a/test/metabase/analytics/metaplow_test.clj
+++ b/test/metabase/analytics/metaplow_test.clj
@@ -1,18 +1,51 @@
(ns metabase.analytics.metaplow-test
(:require
+ [clj-http.client :as http]
[clojure.test :refer :all]
+ [diehard.circuit-breaker :as dh.cb]
[metabase.analytics.metaplow :as metaplow]
[metabase.analytics.settings :as analytics.settings]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
+ [metabase.util.json :as json]
+ [metabase.util.retry :as retry]
[metabase.version.core :as version])
(:import
+ (dev.failsafe CircuitBreaker)
+ (java.net ConnectException)
(java.util.concurrent CountDownLatch TimeUnit)))
(set! *warn-on-reflection* true)
(use-fixtures :once (fixtures/initialize :db))
+(defn- send-stub
+ "Returns `[attempts stub]`, where `stub` stands in for [[metabase.analytics.metaplow/send-event!]] and `attempts`
+ counts the calls to it. `stub` hands back `responses` in order, repeating the last once exhausted, after sleeping
+ `sleep-ms` to stand in for a slow collector. After `limit` calls it switches to a non-retryable status, so a retry
+ policy that has gone unbounded fails an assertion loudly instead of hanging the run."
+ [responses & {:keys [limit sleep-ms] :or {limit 20 sleep-ms 0}}]
+ (let [attempts (atom 0)
+ responses (vec responses)]
+ [attempts (fn [_payload]
+ (let [n (swap! attempts inc)]
+ (when (pos? sleep-ms)
+ (Thread/sleep (long sleep-ms)))
+ (if (> n limit)
+ {:status 400}
+ (nth responses (min (dec n) (dec (count responses)))))))]))
+
+(defmacro ^:private with-fast-retries
+ "Run `body` under the real retry policy with its backoff collapsed, so tests that exhaust retries don't sleep for
+ seconds. The attempt count and the duration ceiling still come from the policy itself.
+
+ Only for sends driven directly from the test thread. `pipeline-blocking` captures its workers' dynamic bindings
+ once, when the first event of the JVM is enqueued, so wrapping a [[metabase.analytics.metaplow/track-event!]] call
+ in this would pin the collapsed backoff for the rest of the run."
+ [& body]
+ `(binding [retry/*test-time-config-hook* #(assoc % :backoff-ms [1 2])]
+ ~@body))
+
(deftest build-payload-test
(testing "Schemas with an `:event` key produce `<schema>.<event>` and the event key is removed from data"
(let [payload (#'metaplow/build-payload :snowplow/dashboard
@@ -55,50 +88,195 @@
(is (=? {:payload {:hostname "anonymous.metabase.com"}}
(#'metaplow/build-payload :snowplow/dashboard {:event :dashboard-created}))))))
+(defn- track-with-collecting-enqueue!
+ "Calls `track-event!` with [[metabase.analytics.metaplow/enqueue!]] stubbed out, and returns
+ `[returned-value enqueued-payloads]`."
+ []
+ (let [collector (atom [])]
+ (mt/with-dynamic-fn-redefs [metaplow/enqueue! (fn [payload]
+ (swap! collector conj payload)
+ true)]
+ [(metaplow/track-event! :snowplow/dashboard {:event :dashboard-created}) @collector])))
+
(deftest tracking-disabled-test
(testing "When metaplow-tracking-enabled is false the event is not enqueued and the call returns false"
- (mt/with-temporary-setting-values [metaplow-url nil]
- (let [collector (atom [])]
- (mt/with-dynamic-fn-redefs [metaplow/enqueue! (fn [payload]
- (swap! collector conj payload)
- true)]
- (is (false? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created})))
- (is (empty? @collector)))))))
+ (testing "- no collector URL resolves"
+ (mt/with-temporary-setting-values [anon-tracking-enabled true
+ metaplow-url nil]
+ (is (= [false []] (track-with-collecting-enqueue!)))))
+ (testing "- anonymous tracking is off, even though a collector URL resolves"
+ (mt/with-temporary-setting-values [anon-tracking-enabled false
+ metaplow-url "http://fake-metaplow/api/send"]
+ (is (= [false []] (track-with-collecting-enqueue!))))))
+ (testing "With anonymous tracking on and a collector URL resolved, the event is enqueued"
+ (mt/with-temporary-setting-values [anon-tracking-enabled true
+ metaplow-url "http://fake-metaplow/api/send"]
+ (let [[returned payloads] (track-with-collecting-enqueue!)]
+ (is (true? returned))
+ (is (=? [{:payload {:name "dashboard.dashboard_created"}}] payloads))))))
+
+(deftest send-event!-test
+ (mt/with-temporary-setting-values [metaplow-url "http://fake-metaplow/api/send"]
+ (testing "The collector's response is returned as-is"
+ (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts] {:status 200 :body "ok"})]
+ (is (=? {:status 200} (#'metaplow/send-event! {:type "event"})))))
+ (testing "A connection failure is caught and reported as status -1"
+ (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts] (throw (ConnectException. "nope")))]
+ (is (= {:status -1} (#'metaplow/send-event! {:type "event"})))))
+ (testing "The payload is POSTed to `metaplow-url` as JSON"
+ (let [calls (atom [])]
+ (mt/with-dynamic-fn-redefs [http/post (fn [url opts] (swap! calls conj [url opts]) {:status 200})]
+ (#'metaplow/send-event! {:type "event" :payload {:name "dashboard.created"}})
+ (let [[[url opts]] @calls]
+ (is (= "http://fake-metaplow/api/send" url))
+ (is (= :json (:content-type opts)))
+ (is (= {"type" "event" "payload" {"name" "dashboard.created"}}
+ (json/decode (:body opts))))))))))
+
+(deftest retry-policy-test
+ (testing "The policy is explicit in code rather than inherited from the retry library's defaults"
+ (is (=? {:max-retries 2
+ :backoff-ms [500 2000]
+ :max-duration-ms 5000}
+ @#'metaplow/retry-policy)))
+ ;; a breaker of its own, so breaker state neither leaks in from another test nor out of this one. Plain
+ ;; `with-redefs`: the var holds a CircuitBreaker, which `mt/with-dynamic-fn-redefs` rejects as a non-fn
+ (with-redefs [metaplow/breaker (#'metaplow/new-breaker nil)]
+ (with-fast-retries
+ (testing "A permanently failing collector is attempted a bounded number of times, then given up on"
+ (let [[attempts stub] (send-stub [{:status -1}])]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]
+ (is (= :error (#'metaplow/send-event-with-retries! {:type "event"})))
+ (is (= 3 @attempts) "one initial attempt plus two retries, and no more"))))
+ (testing "A collector that accepts the event is called exactly once"
+ (let [[attempts stub] (send-stub [{:status 200}])]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]
+ (is (= :sent (#'metaplow/send-event-with-retries! {:type "event"})))
+ (is (= 1 @attempts)))))
+ (testing "A transient failure is retried and the retry's success is reported as `:sent`"
+ (let [[attempts stub] (send-stub [{:status 503} {:status 200}])]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]
+ (is (= :sent (#'metaplow/send-event-with-retries! {:type "event"})))
+ (is (= 2 @attempts)))))
+ (testing "A status the collector will never accept is not retried"
+ (doseq [status [400 401 403 410 422]]
+ (let [[attempts stub] (send-stub [{:status status}])]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]
+ (is (= :error (#'metaplow/send-event-with-retries! {:type "event"})))
+ (is (= 1 @attempts) (str "status " status " should not be retried")))))))
+ (testing "An attempt that outlasts the duration ceiling is not retried, even though retries remain"
+ ;; the real ceiling is 5s and `send-event!`'s own timeouts are 5s apiece, so against a collector that drops
+ ;; packets rather than refusing the connection a failing event gets one attempt, not three
+ ;; `update ... min` rather than `assoc`, so this fails loudly if the policy ever loses its duration ceiling
+ (binding [retry/*test-time-config-hook* #(-> (update % :max-duration-ms min 10)
+ (assoc :backoff-ms [1 2]))]
+ (let [[attempts stub] (send-stub [{:status -1}] :sleep-ms 50)]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]
+ (is (= :error (#'metaplow/send-event-with-retries! {:type "event"})))
+ (is (= 1 @attempts) "the duration ceiling stopped the retries")))))))
+
+(deftest circuit-breaker-test
+ (testing "The breaker policy is explicit in code: a failure threshold, a cooldown, and a success threshold"
+ (is (=? {:failure-threshold 10
+ :delay-ms 60000
+ :success-threshold 1}
+ @#'metaplow/breaker-config)))
+ (with-fast-retries
+ (testing "Ten consecutive failed events open the breaker, and further events are rejected without a send"
+ (let [breaker (#'metaplow/new-breaker nil)
+ [attempts stub] (send-stub [{:status -1}] :limit 200)]
+ (with-redefs [metaplow/breaker breaker]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! stub]
+ (is (= (repeat 10 :error)
+ (vec (repeatedly 10 #(#'metaplow/send-event-with-retries! {:type "event"}))))
+ "the first ten events are attempted and fail")
+ (is (= 30 @attempts) "ten events at three attempts each")
+ (is (= :open (dh.cb/state breaker)))
+ (reset! attempts 0)
+ (is (= :rejected (#'metaplow/send-event-with-retries! {:type "event"})))
+ (is (zero? @attempts) "no send is attempted while the breaker is open")))))
+ (testing "A success closes the breaker again once it half-opens"
+ (let [breaker (#'metaplow/new-breaker nil)]
+ (with-redefs [metaplow/breaker breaker]
+ (let [[_ fail] (send-stub [{:status -1}] :limit 200)]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! fail]
+ (dotimes [_ 10] (#'metaplow/send-event-with-retries! {:type "event"}))
+ (is (= :open (dh.cb/state breaker)))))
+ ;; stand in for the 60-second cooldown elapsing, rather than racing a timer
+ (.halfOpen ^CircuitBreaker breaker)
+ (let [[attempts ok] (send-stub [{:status 200}])]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! ok]
+ (is (= :sent (#'metaplow/send-event-with-retries! {:type "event"}))
+ "the half-open probe is let through")
+ (is (= 1 @attempts))
+ (is (= :closed (dh.cb/state breaker)))))
+ (testing "and a later failure is attempted rather than rejected"
+ (let [[attempts fail] (send-stub [{:status -1}])]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! fail]
+ (is (= :error (#'metaplow/send-event-with-retries! {:type "event"})))
+ (is (= 3 @attempts))))))))))
+
+(deftest error-metrics-test
+ (mt/with-prometheus-system! [_ system]
+ (let [errors (fn [stage]
+ (or (mt/metric-value system :metabase-metaplow/errors {:stage stage}) 0.0))]
+ (testing "A failed outbound attempt is counted against the send stage"
+ (mt/with-temporary-setting-values [metaplow-url "http://fake-metaplow/api/send"]
+ (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts] (throw (ConnectException. "nope")))]
+ (#'metaplow/send-event! {:type "event"})))
+ (is (= 1.0 (errors :send-event!))))
+ (with-fast-retries
+ (let [[_ fail] (send-stub [{:status -1}] :limit 200)]
+ (with-redefs [metaplow/breaker (#'metaplow/new-breaker nil)]
+ (mt/with-dynamic-fn-redefs [metaplow/send-event! fail]
+ (testing "Giving up on an event is counted against the delivery stage"
+ (#'metaplow/send-event-with-retries! {:type "event"})
+ (is (= 1.0 (errors :send-event-with-retries!)))
+ (is (= 0.0 (errors :circuit-breaker-open))))
+ (testing "A breaker rejection is counted under its own stage, distinct from a delivery failure"
+ (dotimes [_ 9] (#'metaplow/send-event-with-retries! {:type "event"}))
+ (is (= 10.0 (errors :send-event-with-retries!)))
+ (#'metaplow/send-event-with-retries! {:type "event"})
+ (is (= 1.0 (errors :circuit-breaker-open)))
+ (is (= 10.0 (errors :send-event-with-retries!))
+ "a rejected event is not also counted as a delivery failure")))))))))
(deftest pipeline-integration-test
(mt/with-temporary-setting-values [metaplow-url "http://fake-metaplow/api/send"
anon-tracking-enabled true]
- (testing "track-event! enqueues onto the real channel and the pipeline worker invokes send-event-with-retries!"
- (let [received (promise)]
- ;; the pipeline's worker threads are spawned once from a defonce delay, so they never inherit a
- ;; test's *local-redefs*; only a root swap reaches them
- #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
- (with-redefs [metaplow/send-event-with-retries! (fn [payload]
- (deliver received payload)
- :sent)]
- (is (true? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id 7})))
- (let [payload (deref received 1000 ::timeout)]
- (is (not= ::timeout payload)
- "Worker did not consume the event within 1s")
- (when (not= ::timeout payload)
- (is (=? {:type "event"
- :payload {:name "dashboard.dashboard_created"
- :data {"dashboard_id" 7}}}
- payload)))))))
- (testing "Sending 200 events: all of them traverse the pipeline"
- (let [received (atom [])
- latch (CountDownLatch. 200)]
- ;; the pipeline's worker threads are spawned once from a defonce delay, so they never inherit a
- ;; test's *local-redefs*; only a root swap reaches them
- #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
- (with-redefs [metaplow/send-event-with-retries! (fn [payload]
- (swap! received conj payload)
- (.countDown latch)
- :sent)]
- (doseq [i (range 1 201)]
- (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id i}))
- (is (true? (.await latch 1 TimeUnit/SECONDS))
- "Pipeline did not process all 200 events within 1s")
- (is (= 200 (count @received)))
- (is (= (set (range 1 201))
- (set (map #(get-in % [:payload :data "dashboard_id"]) @received)))))))))
+ ;; a breaker of its own, so an unrelated test that left the process-wide one open can't make this one flake
+ (with-redefs [metaplow/breaker (#'metaplow/new-breaker nil)]
+ (testing "track-event! enqueues onto the real channel and a pipeline worker POSTs the payload"
+ (let [received (promise)]
+ ;; the pipeline's worker threads are spawned once from a defonce delay, so they never inherit a
+ ;; test's *local-redefs*; only a root swap reaches them
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
+ (with-redefs [metaplow/send-event! (fn [payload]
+ (deliver received payload)
+ {:status 200})]
+ (is (true? (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id 7})))
+ (let [payload (deref received 1000 ::timeout)]
+ (is (not= ::timeout payload)
+ "Worker did not consume the event within 1s")
+ (when (not= ::timeout payload)
+ (is (=? {:type "event"
+ :payload {:name "dashboard.dashboard_created"
+ :data {"dashboard_id" 7}}}
+ payload)))))))
+ (testing "Sending 200 events: all of them traverse the pipeline"
+ (let [received (atom [])
+ latch (CountDownLatch. 200)]
+ ;; the pipeline's worker threads are spawned once from a defonce delay, so they never inherit a
+ ;; test's *local-redefs*; only a root swap reaches them
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
+ (with-redefs [metaplow/send-event! (fn [payload]
+ (swap! received conj payload)
+ (.countDown latch)
+ {:status 200})]
+ (doseq [i (range 1 201)]
+ (metaplow/track-event! :snowplow/dashboard {:event :dashboard-created :dashboard-id i}))
+ (is (true? (.await latch 1 TimeUnit/SECONDS))
+ "Pipeline did not process all 200 events within 1s")
+ (is (= 200 (count @received)))
+ (is (= (set (range 1 201))
+ (set (map #(get-in % [:payload :data "dashboard_id"]) @received))))))))))
diff --git a/test/metabase/analytics/settings_test.clj b/test/metabase/analytics/settings_test.clj
index add6e67abc3..0fadae83022 100644
--- a/test/metabase/analytics/settings_test.clj
+++ b/test/metabase/analytics/settings_test.clj
@@ -2,6 +2,8 @@
(:require
[clojure.test :refer :all]
[metabase.analytics.settings :as analytics.settings]
+ [metabase.config.core :as config]
+ [metabase.premium-features.core :as premium-features]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util.date-2 :as u.date]
@@ -46,3 +48,58 @@
clojure.lang.ExceptionInfo
#"Setting analytics-pii-retention-enabled is not enabled because feature :audit-app is not available"
(analytics.settings/analytics-pii-retention-enabled! true)))))))
+
+(deftest metaplow-url-default-test
+ (testing "In production run mode the collector URL defaults to the Metabase Track collector"
+ (with-redefs [config/is-prod? true]
+ (is (= "https://product-analytics-ingestion.metabase.com/api/send"
+ (#'analytics.settings/default-metaplow-url)))))
+ (testing "In dev, test and e2e run modes there is no default, so those runs send nothing"
+ (with-redefs [config/is-prod? false]
+ (is (nil? (#'analytics.settings/default-metaplow-url))))))
+
+(deftest metaplow-url-is-unset-outside-production-test
+ (testing "There is no default in this run mode, so the setting resolves as unset"
+ (is (nil? (analytics.settings/metaplow-url)))
+ ;; a blank env var value is treated as absent, so this is also what a run with `MB_METAPLOW_URL=` resolves to
+ (mt/with-temp-env-var-value! [mb-metaplow-url ""]
+ (is (nil? (analytics.settings/metaplow-url)))
+ (is (false? (analytics.settings/metaplow-tracking-enabled))))))
+
+(deftest metaplow-url-air-gap-test
+ (mt/with-temporary-setting-values [anon-tracking-enabled true
+ metaplow-url nil]
+ (with-redefs [config/is-prod? true]
+ (testing "In production run mode with nothing set, the Metabase Track collector is the default"
+ (mt/with-dynamic-fn-redefs [premium-features/airgap-enabled (constantly false)]
+ (is (= "https://product-analytics-ingestion.metabase.com/api/send"
+ (analytics.settings/metaplow-url)))
+ (is (true? (analytics.settings/metaplow-tracking-enabled)))))
+ (mt/with-dynamic-fn-redefs [premium-features/airgap-enabled (constantly true)]
+ (testing "An air-gapped instance gets no default collector, so nothing is sent"
+ (is (nil? (analytics.settings/metaplow-url)))
+ (is (false? (analytics.settings/metaplow-tracking-enabled))))
+ (testing "An air-gapped instance still sends to an explicitly set collector"
+ (mt/with-temporary-setting-values [metaplow-url "https://collector.example.com/api/send"]
+ (is (= "https://collector.example.com/api/send"
+ (analytics.settings/metaplow-url)))
+ (is (true? (analytics.settings/metaplow-tracking-enabled)))))
+ (testing "...including one set by env var"
+ (mt/with-temp-env-var-value! [mb-metaplow-url "https://env-collector.example.com/api/send"]
+ (is (= "https://env-collector.example.com/api/send"
+ (analytics.settings/metaplow-url)))
+ (is (true? (analytics.settings/metaplow-tracking-enabled)))))))))
+
+(deftest metaplow-tracking-enabled-test
+ (testing "Metabase Track is on when anonymous tracking is on and a collector URL resolves"
+ (mt/with-temporary-setting-values [anon-tracking-enabled true
+ metaplow-url "https://collector.example.com/api/send"]
+ (is (true? (analytics.settings/metaplow-tracking-enabled))))))
+
+(deftest anon-tracking-opt-out-suppresses-both-destinations-test
+ (testing "Turning anonymous tracking off suppresses Metabase Track as well as Snowplow"
+ (mt/with-temporary-setting-values [anon-tracking-enabled false
+ metaplow-url "https://collector.example.com/api/send"
+ snowplow-available true]
+ (is (false? (analytics.settings/metaplow-tracking-enabled)))
+ (is (false? (analytics.settings/snowplow-enabled))))))
diff --git a/test/metabase/analytics/snowplow_test.clj b/test/metabase/analytics/snowplow_test.clj
index 4737576c362..8bf71c1ea71 100644
--- a/test/metabase/analytics/snowplow_test.clj
+++ b/test/metabase/analytics/snowplow_test.clj
@@ -14,8 +14,11 @@
[toucan2.core :as t2])
(:import
(com.snowplowanalytics.snowplow.tracker.events SelfDescribing)
+ (com.snowplowanalytics.snowplow.tracker.http ApacheHttpClientAdapter)
(com.snowplowanalytics.snowplow.tracker.payload SelfDescribingJson)
- (java.util LinkedHashMap)))
+ (java.net InetAddress ServerSocket)
+ (java.util LinkedHashMap)
+ (org.apache.http.client.config CookieSpecs RequestConfig)))
(set! *warn-on-reflection* true)
@@ -185,3 +188,30 @@
(mt/with-temporary-setting-values [anon-tracking-enabled false]
(analytics/track-event! :snowplow/account {:event :new_instance_created} nil)
(is (= [] (pop-event-data-and-user-id!))))))))))
+
+(deftest ^:parallel http-timeouts-test
+ (testing "the Snowplow HTTP client has finite timeouts"
+ (let [^RequestConfig config (#'snowplow/request-config)]
+ ;; Apache HttpClient reads any non-positive value as "wait forever".
+ (is (pos? (.getConnectTimeout config))
+ "connect timeout must be finite")
+ (is (pos? (.getSocketTimeout config))
+ "socket timeout must be finite")
+ (is (pos? (.getConnectionRequestTimeout config))
+ "connection-request timeout must be finite")
+ (testing "and still sets the cookie policy (PR #24579)"
+ (is (= CookieSpecs/STANDARD (.getCookieSpec config)))))))
+
+(deftest ^:parallel unresponsive-collector-test
+ (testing "a collector that accepts connections but never responds cannot hold an emitter thread indefinitely"
+ ;; A `ServerSocket` nobody ever calls `.accept` on still completes the TCP handshake out of its backlog, so this
+ ;; is a connected socket that will never produce a response.
+ (with-open [blackhole (ServerSocket. 0 1 (InetAddress/getLoopbackAddress))]
+ (let [^ApacheHttpClientAdapter adapter (#'snowplow/http-client-adapter
+ (str "http://127.0.0.1:" (.getLocalPort blackhole)))
+ event (SelfDescribingJson. "iglu:com.metabase/instance/jsonschema/1-1-2" {})
+ response (future (.post adapter event))
+ result (deref response 15000 ::never-returned)]
+ (future-cancel response)
+ (is (= -1 result)
+ "the POST should report a failed request rather than block forever")))))
diff --git a/test/metabase/server/middleware/security_test.clj b/test/metabase/server/middleware/security_test.clj
index c9e4c439533..8b846fe1511 100644
--- a/test/metabase/server/middleware/security_test.clj
+++ b/test/metabase/server/middleware/security_test.clj
@@ -89,6 +89,22 @@
(testing "and script-src-elem specifically is never introduced"
(is (not (contains? (csp-directive-names injection) "script-src-elem")))))))
+(deftest csp-header-analytics-collectors-test
+ (testing "the analytics collectors are in connect-src so the browser legs can post to them"
+ (mt/with-temporary-setting-values [anon-tracking-enabled true
+ metaplow-url "https://product-analytics-ingestion.metabase.com/api/send"
+ snowplow-url "https://sp.metabase.com"]
+ (let [connect-src (csp-directive "connect-src")]
+ (is (str/includes? connect-src "https://product-analytics-ingestion.metabase.com/api/send"))
+ (is (str/includes? connect-src "https://sp.metabase.com")))))
+ (testing "and neither is listed when anonymous tracking is off"
+ (mt/with-temporary-setting-values [anon-tracking-enabled false
+ metaplow-url "https://product-analytics-ingestion.metabase.com/api/send"
+ snowplow-url "https://sp.metabase.com"]
+ (let [connect-src (csp-directive "connect-src")]
+ (is (not (str/includes? connect-src "product-analytics-ingestion.metabase.com")))
+ (is (not (str/includes? connect-src "sp.metabase.com")))))))
+
(deftest csp-header-iframe-hosts-tests
(testing "Allowed iframe hosts setting is used in the CSP frame-src directive."
(mt/with-temporary-setting-values [allowed-iframe-hosts "https://www.wikipedia.org, https://www.typescriptlang.org/ https://clojure.org"]