OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
http: enforce the rate limits the API docs already publish
Carl-Gerhard Lindesvärdd84d1d48fb1c
diff --git a/packages/core/src/http/rate-limit.test.ts b/packages/core/src/http/rate-limit.test.tsnew file mode 100644index 000000000..166a9a47a--- /dev/null+++ b/packages/core/src/http/rate-limit.test.ts@@ -0,0 +1,50 @@+// The prefix table IS the contract: it has to match what+// docs/api-reference/rate-limits.mdx publishes, and the paths that page calls+// unlimited must stay absent from it (ISSUES.md H4).++import { describe, expect, test } from 'bun:test';+import { HTTP_RATE_LIMITS, limitFor } from './rate-limit';++// apps/api/src/routes/*.router.ts on V1, and the published page for the+// first three.+const V1_LIMITS = {+ '/manage/clients': { max: 20, windowMs: 10_000 },+ '/export/events': { max: 100, windowMs: 10_000 },+ '/insights/acme-web/overview': { max: 100, windowMs: 10_000 },+ '/mcp': { max: 60, windowMs: 60_000 },+} as const;++describe('HTTP rate limits', () => {+ test("each documented surface keeps V1's numbers", () => {+ for (const [path, expected] of Object.entries(V1_LIMITS)) {+ const limit = limitFor(path);+ expect([path, limit?.max, limit?.windowMs]).toEqual([+ path,+ expected.max,+ expected.windowMs,+ ]);+ }+ });++ // The same page promises these are unlimited. A prefix added carelessly+ // here would start throttling ingestion.+ test('the ingestion paths are not limited', () => {+ for (const path of [+ '/track',+ '/track/device-id',+ '/profile',+ '/profile/increment',+ '/import/events',+ '/healthcheck',+ '/metrics',+ ]) {+ expect([path, limitFor(path)]).toEqual([path, undefined]);+ }+ });++ test('every entry names a scope, so buckets cannot collide', () => {+ const scopes = HTTP_RATE_LIMITS.map((limit) => limit.scope);+ expect(new Set(scopes).size).toBe(scopes.length);+ expect(scopes.every(Boolean)).toBe(true);+ });+});diff --git a/packages/core/src/http/rate-limit.ts b/packages/core/src/http/rate-limit.tsnew file mode 100644index 000000000..22bb29740--- /dev/null+++ b/packages/core/src/http/rate-limit.ts@@ -0,0 +1,145 @@+// The HTTP limiter, ported from apps/api/src/utils/rate-limiter.ts (V1),+// which used @fastify/rate-limit. Deliberately simpler than the tRPC one in+// rpc/rate-limit.ts: a fixed window with no escalation, because that is what+// V1 ran and what docs/api-reference/rate-limits.mdx publishes.+//+// The rewrite shipped without it, so the limits that page advertises were not+// enforced anywhere (ISSUES.md H4).++import { getRedisCache } from '@openpanel/redis';+import { Elysia } from 'elysia';+import type { AppDeps } from '../context';+import type { Logger } from '../logger';+import { getTrustedIpFromHeaders } from '../shared/get-client-ip';+import { requestContext } from './context';++const TOO_MANY_REQUESTS = 429;+const KEY_MISSING = -2;+const KEY_WITHOUT_EXPIRY = -1;++/** V1's body, byte for byte. */+const TOO_MANY_REQUESTS_BODY = {+ statusCode: TOO_MANY_REQUESTS,+ error: 'Too Many Requests',+ message: 'You have exceeded the rate limit for this endpoint.',+} as const;++/**+ * As in V1: the API client if there is one, otherwise the TRUSTED ip.+ * `x-client-ip` and `x-forwarded-for` are caller-set, so keying on the+ * attribution address would hand out a fresh bucket per request.+ */+const TRUSTED_ONLY = { attributionOrder: undefined, trustedOrder: undefined };++function fingerprintFor(headers: Headers, remoteAddress?: string): string {+ const clientId = headers.get('openpanel-client-id');+ if (clientId) {+ return `client:${clientId}`;+ }+ const { ip } = getTrustedIpFromHeaders(TRUSTED_ONLY, headers, remoteAddress);+ // Fail closed: an edge that stops forwarding IPs should throttle, not open.+ return `ip:${ip || 'unknown'}`;+}++export interface HttpRateLimit {+ /** Requests allowed per window. */+ max: number;+ windowMs: number;+ /** Groups every route under one bucket, as V1's per-router plugin did. */+ scope: string;+}++/**+ * `true` when the caller is over its limit and the request must be refused.+ * A Redis failure allows the request: this guards a documented quota, not a+ * credential, and V1 behaved the same way.+ */+export async function isRateLimited(+ { max, windowMs, scope }: HttpRateLimit,+ headers: Headers,+ remoteAddress: string | undefined,+ logger: Logger+): Promise<boolean> {+ const key = `rl:http:${scope}:${fingerprintFor(headers, remoteAddress)}`;++ try {+ const redis = getRedisCache();+ const results = await redis.pipeline().incr(key).pttl(key).exec();+ const hits = Number(results?.[0]?.[1] ?? 1);+ const ttlMs = Number(results?.[1]?.[1] ?? KEY_MISSING);++ // The window opens on the first hit. Re-arm a counter that lost its+ // expiry, or it would count up forever and lock the caller out for good.+ if (hits === 1 || ttlMs === KEY_WITHOUT_EXPIRY || ttlMs === KEY_MISSING) {+ await redis.pexpire(key, windowMs);+ }++ if (hits > max) {+ logger.warn({ scope, key, hits, max, windowMs }, 'rate limit exceeded');+ return true;+ }+ return false;+ } catch (error) {+ logger.error({ err: error, scope }, 'rate limit store unavailable');+ return false;+ }+}++/**+ * V1's limits, router for router (apps/api/src/routes/*.router.ts). The first+ * matching prefix wins, so order matters only where one prefix contains+ * another — none do today.+ *+ * `/export`, `/insights` and `/manage` are what+ * docs/api-reference/rate-limits.mdx publishes. `/track`, `/profile` and+ * `/import` are absent on purpose: the same page promises they are unlimited.+ */+export const HTTP_RATE_LIMITS: readonly (HttpRateLimit & {+ prefix: string;+})[] = [+ { prefix: '/manage', scope: 'manage', max: 20, windowMs: 10_000 },+ { prefix: '/export', scope: 'export', max: 100, windowMs: 10_000 },+ { prefix: '/insights', scope: 'insights', max: 100, windowMs: 10_000 },+ { prefix: '/mcp', scope: 'mcp', max: 60, windowMs: 60_000 },+];++export function limitFor(pathname: string): HttpRateLimit | undefined {+ return HTTP_RATE_LIMITS.find((limit) => pathname.startsWith(limit.prefix));+}++/**+ * Mounted once per route surface rather than per module, so a new `/manage`+ * route cannot quietly arrive without a limit. V1 registered the plugin per+ * router; the prefix table above is that same mapping in one place.+ */+export function httpRateLimit(deps: AppDeps) {+ return new Elysia({ name: 'core/http/rate-limit' })+ .use(requestContext(deps))+ .onBeforeHandle({ as: 'global' }, async ({ ctx, request, path, set }) => {+ const limit = limitFor(path);+ if (!limit) {+ return;+ }+ // `ctx` is absent for a request that matched no route, and an unmatched+ // path deserves its 404 rather than a quota.+ if (!ctx) {+ return;+ }+ // No socket address at this layer — `HttpCtx` carries none, and the+ // tRPC context passes `undefined` too (rpc/base.ts). The trusted+ // headers are the fingerprint; without one, callers share the+ // fail-closed `unknown` bucket.+ const limited = await isRateLimited(+ limit,+ request.headers,+ undefined,+ ctx.logger+ );+ if (limited) {+ set.status = TOO_MANY_REQUESTS;+ return TOO_MANY_REQUESTS_BODY;+ }+ });+}++export { TOO_MANY_REQUESTS, TOO_MANY_REQUESTS_BODY };diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.tsindex 8a20d0907..ba0e29cc0 100644--- a/packages/core/src/rest.routes.ts+++ b/packages/core/src/rest.routes.ts@@ -9,6 +9,7 @@ import { Elysia } from 'elysia';import type { AppDeps } from './context';import { createDebugRoutes } from './http/debug.routes';import { httpMetrics } from './http/http.metrics';+import { httpRateLimit } from './http/rate-limit';import { queues } from './jobs.registry';import { registry } from './metrics';import { assistantRoutes } from './modules/assistant/assistant.routes';@@ -68,6 +69,10 @@ const METRICS_EXPOSITION_TTL_MS = 5000;// the one route surface whose hook ORDER is part of the contract.export const publicApiRoutes = (deps: AppDeps) =>new Elysia({ name: 'core/public-api-routes' })+ // Mounted on the SURFACE, not per module, so a new /manage or /export+ // route cannot arrive without the limit the docs advertise. The prefix+ // table is in http/rate-limit.ts.+ .use(httpRateLimit(deps)).use(ingestRoutes(deps)).use(importRoutes(deps)).use(projectRoutes(deps))@@ -96,6 +101,8 @@ export const publicApiRoutes = (deps: AppDeps) =>// misc's `/misc/*` (M7-008) joined it too — `/misc` IS in `corsPaths`.export const dashboardRoutes = (deps: AppDeps) =>new Elysia({ name: 'core/dashboard-routes' })+ // For `/mcp`; every other path on this surface is unmatched by the table.+ .use(httpRateLimit(deps)).use(gscRoutes(deps)).use(assistantRoutes(deps)).use(mcpRoutes(deps))