Rallly is an open-source scheduling and collaboration tool designed to make organizing events and meetings easier.
TypeScript i18nextnext-authnextjspostgresqlprismareact-emailt3-stacktailwindcsstrpcturborepotypescriptzod
🔔 Report the API rate limiter failing closed to Sentry and ship wide events as logs (#3265)
Luke Vella25eb0bdbf2fc
diff --git a/apps/web/sentry.server.config.ts b/apps/web/sentry.server.config.tsindex bb30a4c29..9371a94b8 100644--- a/apps/web/sentry.server.config.ts+++ b/apps/web/sentry.server.config.ts@@ -17,6 +17,23 @@ Sentry.init({// Don't send personal identifiable information (PII) to Sentry.sendDefaultPii: false,+ // Ship the 4xx/5xx wide events as structured logs so rate limiter usage+ // (`rateLimiterDailyConsumedPoints`, 429s per `spaceId`) is queryable and+ // alertable for 30 days. Info-level events are every request; they stay in+ // Vercel to keep the log quota for the ones worth searching.+ enableLogs: true,+ integrations: [+ Sentry.pinoIntegration({ log: { levels: ["warn", "error", "fatal"] } }),+ ],+ beforeSendLog(log) {+ // `sendDefaultPii` does not cover custom log attributes.+ if (log.attributes) {+ const { ip, ...attributes } = log.attributes;+ return { ...log, attributes };+ }+ return log;+ },+// Uncomment the line below to enable Spotlight (https://spotlightjs.com)// spotlight: process.env.NODE_ENV === 'development',});diff --git a/apps/web/src/app/api/middleware/rate-limit.ts b/apps/web/src/app/api/middleware/rate-limit.tsindex f8b4837f8..58957c9fd 100644--- a/apps/web/src/app/api/middleware/rate-limit.ts+++ b/apps/web/src/app/api/middleware/rate-limit.ts@@ -1,3 +1,4 @@+import * as Sentry from "@sentry/nextjs";import type { Context } from "hono";import { createMiddleware } from "hono/factory";import {@@ -36,6 +37,26 @@ export type RateLimitFailure = {message: string;};+/**+ * The fail-closed 503 is returned, not thrown, so nothing upstream reports it.+ * A fixed message per reason keeps Sentry grouping to one issue per cause;+ * the store's own message travels as `cause` and on the wide event.+ */+export class RateLimitStoreError extends Error {+ readonly reason: RateLimitFailure["reason"];++ constructor(reason: RateLimitFailure["reason"], options?: ErrorOptions) {+ super(+ reason === "store_unavailable"+ ? "Rate limit store is not configured"+ : "Rate limit store request failed",+ options,+ );+ this.name = "RateLimitStoreError";+ this.reason = reason;+ }+}+type RateLimitEnv = {Variables: {apiAuth: {@@ -181,8 +202,38 @@ function setRateLimitHeaders(c: Context, info: RateLimitInfo) {c.header("RateLimit-Reset", resetSeconds(binding).toString());}-function serviceUnavailable(c: Context, failure: RateLimitFailure) {+const STORE_ERROR_REPORT_INTERVAL_MS = 60 * 1000;+const lastStoreErrorReportAt = new Map<RateLimitFailure["reason"], number>();++/**+ * During an outage every API request takes this path, so report at most one+ * event per reason per minute per process. The alert only needs the first;+ * the wide event keeps the per-request detail.+ */+function reportStoreError(+ failure: RateLimitFailure,+ { spaceId, cause }: { spaceId: string; cause?: unknown },+) {+ const now = Date.now();+ const lastReportAt = lastStoreErrorReportAt.get(failure.reason);+ if (lastReportAt && now - lastReportAt < STORE_ERROR_REPORT_INTERVAL_MS) {+ return;+ }+ lastStoreErrorReportAt.set(failure.reason, now);+ Sentry.captureException(new RateLimitStoreError(failure.reason, { cause }), {+ tags: { rateLimiterError: failure.reason },+ fingerprint: ["rate-limit-store", failure.reason],+ extra: { spaceId },+ });+}++function serviceUnavailable(+ c: Context,+ failure: RateLimitFailure,+ { spaceId, cause }: { spaceId: string; cause?: unknown },+) {c.set("rateLimitFailure", failure);+ reportStoreError(failure, { spaceId, cause });return c.json(apiError("SERVICE_UNAVAILABLE",@@ -207,21 +258,30 @@ function serviceUnavailable(c: Context, failure: RateLimitFailure) {* the wide event via `rateLimitFailure`.*/export const rateLimit = createMiddleware<RateLimitEnv>(async (c, next) => {+ const { spaceId } = c.get("apiAuth");if (!store) {- return serviceUnavailable(c, {- reason: "store_unavailable",- message: "Rate limit store is not configured",- });+ return serviceUnavailable(+ c,+ {+ reason: "store_unavailable",+ message: "Rate limit store is not configured",+ },+ { spaceId },+ );}let result: Counters;try {- result = await store.increment(c.get("apiAuth").spaceId);+ result = await store.increment(spaceId);} catch (error) {- return serviceUnavailable(c, {- reason: "store_error",- message: error instanceof Error ? error.message : String(error),- });+ return serviceUnavailable(+ c,+ {+ reason: "store_error",+ message: error instanceof Error ? error.message : String(error),+ },+ { spaceId, cause: error },+ );}const [minuteHits, minuteTtl, dayHits, dayTtl] = result;