Skip to content

Rallly is an open-source scheduling and collaboration tool designed to make organizing events and meetings easier.

🔔 Report the API rate limiter failing closed to Sentry and ship wide events as logs (#3265)

Luke Vella25eb0bdbf2fc
diff --git a/apps/web/sentry.server.config.ts b/apps/web/sentry.server.config.ts
index bb30a4c29..9371a94b8 100644
--- a/apps/web/sentry.server.config.ts
+++ b/apps/web/sentry.server.config.ts
@@ -17,6 +17,23 @@ Sentry.init({
// Don't send personal identifiable information (PII) to Sentry.
sendDefaultPii: false,
+ // Ship the 4xx/5xx wide events as structured logs so rate limiter usage
+ // (`rateLimiterDailyConsumedPoints`, 429s per `spaceId`) is queryable and
+ // alertable for 30 days. Info-level events are every request; they stay in
+ // Vercel to keep the log quota for the ones worth searching.
+ enableLogs: true,
+ integrations: [
+ Sentry.pinoIntegration({ log: { levels: ["warn", "error", "fatal"] } }),
+ ],
+ beforeSendLog(log) {
+ // `sendDefaultPii` does not cover custom log attributes.
+ if (log.attributes) {
+ const { ip, ...attributes } = log.attributes;
+ return { ...log, attributes };
+ }
+ return log;
+ },
+
// Uncomment the line below to enable Spotlight (https://spotlightjs.com)
// spotlight: process.env.NODE_ENV === 'development',
});
diff --git a/apps/web/src/app/api/middleware/rate-limit.ts b/apps/web/src/app/api/middleware/rate-limit.ts
index f8b4837f8..58957c9fd 100644
--- a/apps/web/src/app/api/middleware/rate-limit.ts
+++ b/apps/web/src/app/api/middleware/rate-limit.ts
@@ -1,3 +1,4 @@
+import * as Sentry from "@sentry/nextjs";
import type { Context } from "hono";
import { createMiddleware } from "hono/factory";
import {
@@ -36,6 +37,26 @@ export type RateLimitFailure = {
message: string;
};
+/**
+ * The fail-closed 503 is returned, not thrown, so nothing upstream reports it.
+ * A fixed message per reason keeps Sentry grouping to one issue per cause;
+ * the store's own message travels as `cause` and on the wide event.
+ */
+export class RateLimitStoreError extends Error {
+ readonly reason: RateLimitFailure["reason"];
+
+ constructor(reason: RateLimitFailure["reason"], options?: ErrorOptions) {
+ super(
+ reason === "store_unavailable"
+ ? "Rate limit store is not configured"
+ : "Rate limit store request failed",
+ options,
+ );
+ this.name = "RateLimitStoreError";
+ this.reason = reason;
+ }
+}
+
type RateLimitEnv = {
Variables: {
apiAuth: {
@@ -181,8 +202,38 @@ function setRateLimitHeaders(c: Context, info: RateLimitInfo) {
c.header("RateLimit-Reset", resetSeconds(binding).toString());
}
-function serviceUnavailable(c: Context, failure: RateLimitFailure) {
+const STORE_ERROR_REPORT_INTERVAL_MS = 60 * 1000;
+const lastStoreErrorReportAt = new Map<RateLimitFailure["reason"], number>();
+
+/**
+ * During an outage every API request takes this path, so report at most one
+ * event per reason per minute per process. The alert only needs the first;
+ * the wide event keeps the per-request detail.
+ */
+function reportStoreError(
+ failure: RateLimitFailure,
+ { spaceId, cause }: { spaceId: string; cause?: unknown },
+) {
+ const now = Date.now();
+ const lastReportAt = lastStoreErrorReportAt.get(failure.reason);
+ if (lastReportAt && now - lastReportAt < STORE_ERROR_REPORT_INTERVAL_MS) {
+ return;
+ }
+ lastStoreErrorReportAt.set(failure.reason, now);
+ Sentry.captureException(new RateLimitStoreError(failure.reason, { cause }), {
+ tags: { rateLimiterError: failure.reason },
+ fingerprint: ["rate-limit-store", failure.reason],
+ extra: { spaceId },
+ });
+}
+
+function serviceUnavailable(
+ c: Context,
+ failure: RateLimitFailure,
+ { spaceId, cause }: { spaceId: string; cause?: unknown },
+) {
c.set("rateLimitFailure", failure);
+ reportStoreError(failure, { spaceId, cause });
return c.json(
apiError(
"SERVICE_UNAVAILABLE",
@@ -207,21 +258,30 @@ function serviceUnavailable(c: Context, failure: RateLimitFailure) {
* the wide event via `rateLimitFailure`.
*/
export const rateLimit = createMiddleware<RateLimitEnv>(async (c, next) => {
+ const { spaceId } = c.get("apiAuth");
if (!store) {
- return serviceUnavailable(c, {
- reason: "store_unavailable",
- message: "Rate limit store is not configured",
- });
+ return serviceUnavailable(
+ c,
+ {
+ reason: "store_unavailable",
+ message: "Rate limit store is not configured",
+ },
+ { spaceId },
+ );
}
let result: Counters;
try {
- result = await store.increment(c.get("apiAuth").spaceId);
+ result = await store.increment(spaceId);
} catch (error) {
- return serviceUnavailable(c, {
- reason: "store_error",
- message: error instanceof Error ? error.message : String(error),
- });
+ return serviceUnavailable(
+ c,
+ {
+ reason: "store_error",
+ message: error instanceof Error ? error.message : String(error),
+ },
+ { spaceId, cause: error },
+ );
}
const [minuteHits, minuteTtl, dayHits, dayTtl] = result;