Skip to content

Open-source status page, incident response and uptime monitoring

analytics: throttle event api (#2888)

Thibault Le Ouay7fb1ec27c0ad
diff --git a/apps/server/src/libs/api-request-sampling.ts b/apps/server/src/libs/api-request-sampling.ts
new file mode 100644
index 000000000..d6de4348a
--- /dev/null
+++ b/apps/server/src/libs/api-request-sampling.ts
@@ -0,0 +1,19 @@
+import type { CliInfo } from "@/libs/cli-telemetry";
+
+/** Share of non-CLI `api_request` events sent to OpenPanel. */
+export const API_REQUEST_SAMPLE_RATE = 0.1;
+
+/**
+ * The sample rate to stamp on this request's `api_request` event, or
+ * `undefined` when the event is dropped. CLI requests are always kept so CLI
+ * usage stays exact; everything else is sampled at `API_REQUEST_SAMPLE_RATE`.
+ * Volumes are recovered by weighting each event by `1 / sampleRate`.
+ */
+export function apiRequestSampleRate(
+ cli: CliInfo | undefined,
+): number | undefined {
+ if (cli) return 1;
+ return Math.random() < API_REQUEST_SAMPLE_RATE
+ ? API_REQUEST_SAMPLE_RATE
+ : undefined;
+}
diff --git a/apps/server/src/libs/middlewares/track.test.ts b/apps/server/src/libs/middlewares/track.test.ts
index bf21cb126..6c735b8dc 100644
--- a/apps/server/src/libs/middlewares/track.test.ts
+++ b/apps/server/src/libs/middlewares/track.test.ts
@@ -1,10 +1,13 @@
import { Events } from "@openstatus/analytics";
import type { Workspace } from "@openstatus/db/src/schema";
import {
+ afterEach,
beforeEach,
describe,
expect,
type mock,
+ type Stub,
+ stub,
test,
} from "@openstatus/test-utils";
import { Hono } from "hono";
@@ -63,9 +66,17 @@ function cliCommandCalls() {
}
describe("apiTrackMiddleware", () => {
+ let random: Stub<Math, [], number>;
+
beforeEach(() => {
mockSetupAnalytics.mockClear();
mockTrack.mockClear();
+ // Keep every sampled request unless a test says otherwise.
+ random = stub(Math, "random", () => 0);
+ });
+
+ afterEach(() => {
+ random.restore();
});
test("fires api_request for GET requests with the route pattern", async () => {
@@ -81,9 +92,39 @@ describe("apiTrackMiddleware", () => {
service: "v1",
method: "GET /monitor/:id",
success: false,
+ sampleRate: 0.1,
});
});
+ test("drops non-CLI requests outside the sample", async () => {
+ random.restore();
+ random = stub(Math, "random", () => 0.5);
+
+ const res = await makeApp().request("/whoami", {
+ headers: { "user-agent": "curl/8" },
+ });
+ await flush();
+
+ expect(res.status).toBe(200);
+ expect(mockSetupAnalytics).not.toHaveBeenCalled();
+ expect(mockTrack).not.toHaveBeenCalled();
+ });
+
+ test("always tracks CLI requests", async () => {
+ random.restore();
+ random = stub(Math, "random", () => 0.99);
+
+ await makeApp().request("/whoami", {
+ headers: cliHeaders(crypto.randomUUID()),
+ });
+ await flush();
+
+ expect(mockTrack).toHaveBeenCalledWith(
+ expect.objectContaining({ ...Events.ApiRequest, sampleRate: 1 }),
+ );
+ expect(cliCommandCalls()).toHaveLength(1);
+ });
+
test("skips requests that match no route", async () => {
const res = await makeApp().request("/nope", {
headers: { "user-agent": "curl/8" },
@@ -119,6 +160,7 @@ describe("apiTrackMiddleware", () => {
service: "v1",
method: "GET /whoami",
success: true,
+ sampleRate: 1,
cliCommand: "whoami",
cliVersion: "1.3.2",
});
diff --git a/apps/server/src/libs/middlewares/track.ts b/apps/server/src/libs/middlewares/track.ts
index 628f48d43..784b6819b 100644
--- a/apps/server/src/libs/middlewares/track.ts
+++ b/apps/server/src/libs/middlewares/track.ts
@@ -9,6 +9,7 @@ import type { Context, Next } from "hono";
import { matchedRoutes } from "hono/route";
import { apiAnalyticsIdentity } from "@/libs/analytics-identity";
+import { apiRequestSampleRate } from "@/libs/api-request-sampling";
import { parseCliHeaders, trackCliCommand } from "@/libs/cli-telemetry";
import type { Variables } from "@/types";
@@ -53,7 +54,8 @@ export function trackMiddleware(event: EventProps, eventProps?: string[]) {
* pattern (`GET /v1/monitor/:id`), never the raw URL, to keep it low-cardinality.
*
* Requests from the openstatus CLI also carry `cliCommand`/`cliVersion`, and
- * the first request of each CLI run fires one `cli_command`.
+ * the first request of each CLI run fires one `cli_command`. CLI requests are
+ * always tracked; the rest are sampled (see `apiRequestSampleRate`).
*
* Mount after `authMiddleware`; requests it rejects carry no workspace and are
* skipped, as are requests that match no route. Per-route domain events stay with `trackMiddleware`.
@@ -72,6 +74,9 @@ export function apiTrackMiddleware() {
if (!route) return;
const cli = parseCliHeaders(c.req.raw.headers);
+ const sampleRate = apiRequestSampleRate(cli);
+ if (sampleRate === undefined) return;
+
const success = c.res.status.toString().startsWith("2") && !c.error;
setupAnalytics(apiAnalyticsIdentity(workspace, c.req.raw.headers))
@@ -82,6 +87,7 @@ export function apiTrackMiddleware() {
service: "v1",
method: `${c.req.method} ${route.path}`,
success,
+ sampleRate,
...(cli ? { cliCommand: cli.command } : {}),
...(cli?.version ? { cliVersion: cli.version } : {}),
}),
diff --git a/apps/server/src/routes/rpc/interceptors/__tests__/tracking.test.ts b/apps/server/src/routes/rpc/interceptors/__tests__/tracking.test.ts
index baa056999..b803671ec 100644
--- a/apps/server/src/routes/rpc/interceptors/__tests__/tracking.test.ts
+++ b/apps/server/src/routes/rpc/interceptors/__tests__/tracking.test.ts
@@ -4,10 +4,13 @@ import { MonitorService } from "@openstatus/proto/monitor/v1";
// @ts-nocheck — ConnectRPC's deep generic types (AnyFn, UnaryResponse, etc.)
// are incompatible with the test mocks. All runtime behavior is correct.
import {
+ afterEach,
beforeEach,
describe,
expect,
mock,
+ type Stub,
+ stub,
test,
} from "@openstatus/test-utils";
@@ -84,9 +87,17 @@ function createMockRequest(
}
describe("trackingInterceptor", () => {
+ let random: Stub<Math, [], number>;
+
beforeEach(() => {
mockSetupAnalytics.mockClear();
mockTrack.mockClear();
+ // Keep every sampled request unless a test says otherwise.
+ random = stub(Math, "random", () => 0);
+ });
+
+ afterEach(() => {
+ random.restore();
});
test("tracks mapped event on success", async () => {
@@ -121,6 +132,7 @@ describe("trackingInterceptor", () => {
service: "openstatus.monitor.v1.MonitorService",
method: "DeleteMonitor",
success: true,
+ sampleRate: 0.1,
});
expect(mockTrack).toHaveBeenCalledWith({
...Events.DeleteMonitor,
@@ -184,6 +196,7 @@ describe("trackingInterceptor", () => {
service: "openstatus.monitor.v1.MonitorService",
method: "ListMonitors",
success: true,
+ sampleRate: 0.1,
});
});
@@ -205,6 +218,7 @@ describe("trackingInterceptor", () => {
service: "openstatus.monitor.v1.MonitorService",
method: "DeleteMonitor",
success: false,
+ sampleRate: 0.1,
});
});
@@ -233,6 +247,7 @@ describe("trackingInterceptor", () => {
service: "openstatus.monitor.v1.MonitorService",
method: "ListMonitors",
success: true,
+ sampleRate: 1,
cliCommand: "monitors list",
cliVersion: "1.3.2",
});
@@ -250,6 +265,66 @@ describe("trackingInterceptor", () => {
});
});
+ test("drops non-CLI api_request outside the sample", async () => {
+ random.restore();
+ random = stub(Math, "random", () => 0.5);
+
+ const interceptor = trackingInterceptor();
+ const req = createMockRequest(
+ "openstatus.monitor.v1.MonitorService",
+ "ListMonitors",
+ );
+
+ await interceptor(mockNext({}))(req as never);
+ await flush();
+
+ expect(mockSetupAnalytics).not.toHaveBeenCalled();
+ expect(mockTrack).not.toHaveBeenCalled();
+ });
+
+ test("keeps the domain event when api_request is sampled out", async () => {
+ random.restore();
+ random = stub(Math, "random", () => 0.5);
+
+ const interceptor = trackingInterceptor();
+ const req = createMockRequest(
+ "openstatus.monitor.v1.MonitorService",
+ "DeleteMonitor",
+ );
+
+ await interceptor(mockNext({}))(req as never);
+ await flush();
+
+ expect(mockTrack).toHaveBeenCalledTimes(1);
+ expect(mockTrack).toHaveBeenCalledWith({ ...Events.DeleteMonitor });
+ });
+
+ test("always tracks CLI api_request", async () => {
+ random.restore();
+ random = stub(Math, "random", () => 0.99);
+
+ const interceptor = trackingInterceptor();
+ const req = createMockRequest(
+ "openstatus.monitor.v1.MonitorService",
+ "ListMonitors",
+ {},
+ {
+ headers: {
+ "user-agent": "openstatus-cli/v1.3.2 (darwin; arm64)",
+ "x-openstatus-cli-command": "monitors list",
+ "x-openstatus-cli-invocation": crypto.randomUUID(),
+ },
+ },
+ );
+
+ await interceptor(mockNext({}))(req as never);
+ await flush();
+
+ expect(mockTrack).toHaveBeenCalledWith(
+ expect.objectContaining({ ...Events.ApiRequest, sampleRate: 1 }),
+ );
+ });
+
test("skips tracking when no RPC context", async () => {
const interceptor = trackingInterceptor();
const req = createMockRequest(
diff --git a/apps/server/src/routes/rpc/interceptors/tracking.ts b/apps/server/src/routes/rpc/interceptors/tracking.ts
index 82438545e..7dba19698 100644
--- a/apps/server/src/routes/rpc/interceptors/tracking.ts
+++ b/apps/server/src/routes/rpc/interceptors/tracking.ts
@@ -8,6 +8,7 @@ import {
} from "@openstatus/analytics";
import { apiAnalyticsIdentity } from "../../../libs/analytics-identity";
+import { apiRequestSampleRate } from "../../../libs/api-request-sampling";
import { parseCliHeaders, trackCliCommand } from "../../../libs/cli-telemetry";
import { RPC_CONTEXT_KEY } from "./auth";
@@ -171,6 +172,8 @@ export const RPC_EVENT_MAP: Record<string, RpcEventMapping> = {
*
* Requests from the openstatus CLI also carry `cliCommand`/`cliVersion` on
* `api_request`, and the first request of each CLI run fires one `cli_command`.
+ * CLI requests always fire `api_request`; the rest are sampled (see
+ * `apiRequestSampleRate`). Domain events are never sampled.
*
* Must be placed after authInterceptor (needs workspace context) and
* validationInterceptor (requests it rejects never reach here, so they are not
@@ -209,17 +212,21 @@ function trackRpcCall(req: TrackedRequest, success: boolean) {
const key = `${req.service.typeName}/${req.method.name}`;
const mapping = success ? RPC_EVENT_MAP[key] : undefined;
const cli = parseCliHeaders(req.header);
+ const sampleRate = apiRequestSampleRate(cli);
- const events: (EventProps & Record<string, unknown>)[] = [
- {
+ const events: (EventProps & Record<string, unknown>)[] = [];
+
+ if (sampleRate !== undefined) {
+ events.push({
...Events.ApiRequest,
service: req.service.typeName,
method: req.method.name,
success,
+ sampleRate,
...(cli ? { cliCommand: cli.command } : {}),
...(cli?.version ? { cliVersion: cli.version } : {}),
- },
- ];
+ });
+ }
if (mapping) {
const input = mapping.normalizeInput?.(req.message) ?? req.message;
@@ -227,6 +234,8 @@ function trackRpcCall(req: TrackedRequest, success: boolean) {
events.push({ ...additionalProps, ...mapping.event });
}
+ if (events.length === 0) return;
+
setupAnalytics(apiAnalyticsIdentity(rpcCtx.workspace, req.header))
.then((analytics) =>
Promise.all([
diff --git a/deno.lock b/deno.lock
index c4704b002..5cb6c59e7 100644
--- a/deno.lock
+++ b/deno.lock
@@ -58,6 +58,272 @@
"dependencies": [
]
+ },
+ "members": {
+ "apps/dashboard": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "apps/server": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "apps/status-page": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "apps/web": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/ai": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/api": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/assertions": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/db": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/emails": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/header-analysis": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/importers": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/base": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/bird-whatsapp": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/discord": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/google-chat": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/grafana-oncall": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/ms-teams": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/ntfy": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/opsgenie": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/pagerduty": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/slack": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/telegram": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/twillio-sms": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/notifications/webhook": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/services": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/status-fetcher": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/subscriptions": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/test-utils": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/theme-store": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/tinybird": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/tracker": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/upstash": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ },
+ "packages/utils": {
+ "packageJson": {
+ "dependencies": [
+ "npm:@jsr/std__expect@^1.0.19",
+ "npm:@jsr/std__testing@^1.0.19"
+ ]
+ }
+ }
}
}
}