Skip to content

Open-source status page, incident response and uptime monitoring

12.1 KB
Raw Blame
import { type Context, Hono } from "hono";
import { requestId } from "hono/request-id";
import { describe, expect, test, vi } from "vitest";

import { createRateLimit, credentialKey } from "./rate-limit";

type Env = { Variables: { event: Record<string, unknown> } };

const HUGE = 1_000_000;

// Limiters own an interval timer; build them at module scope, outside the sanitizer.
function build(
  config: {
    perMinute?: number;
    burstPer10s?: number;
    writesPerMinute?: number;
    publicPerMinute?: number;
  },
  handler: (c: Context<Env>) => Response = (c) => c.text("ok"),
) {
  const app = new Hono<Env>();
  const events: Record<string, unknown>[] = [];
  let handled = 0;
  app.use("*", requestId());
  app.use("*", async (c, next) => {
    const event: Record<string, unknown> = {};
    events.push(event);
    c.set("event", event);
    await next();
  });
  app.use(
    "*",
    ...createRateLimit({
      perMinute: HUGE,
      burstPer10s: HUGE,
      writesPerMinute: HUGE,
      publicPerMinute: HUGE,
      ...config,
    }),
  );
  app.all("*", (c) => {
    handled++;
    return handler(c);
  });
  return { app, events, handled: () => handled };
}

const perMinute = build({ perMinute: 600 });
const burst = build({ burstPer10s: 3 });
const writes = build({ writesPerMinute: 2 });
const pub = build({ publicPerMinute: 2 });
const recovery = build({ burstPer10s: 1, perMinute: 2 });
const precedence = build({ burstPer10s: 1 });
const isolation = build({ perMinute: 1, publicPerMinute: 1 });
// stands in for the auth middleware: `valid*` credentials are accepted
const authFailures = build({ perMinute: 2 }, (c) =>
  c.req.header("x-openstatus-key")?.startsWith("valid")
    ? c.text("ok")
    : c.text("nope", 401),
);
const keyApp = new Hono().get("/k", async (c) =>
  c.text(await credentialKey(c)),
);

const keyHeaders = (key: string) => ({ "x-openstatus-key": key });

async function fire(
  app: Hono<Env>,
  path: string,
  n: number,
  init?: RequestInit,
) {
  let last: Response | undefined;
  for (let i = 0; i < n; i++) last = await app.request(path, init);
  return last as Response;
}

describe("rate limit", () => {
  test("601st request in a window from one key gets 429", async () => {
    const { app, events } = perMinute;
    const ok = await fire(app, "/v1/monitor", 600, {
      headers: keyHeaders("a"),
    });
    expect(ok.status).toBe(200);

    const limited = await app.request("/v1/monitor", {
      headers: keyHeaders("a"),
    });
    expect(limited.status).toBe(429);
    expect(Number(limited.headers.get("retry-after"))).toBeGreaterThan(0);
    expect(await limited.json()).toMatchObject({
      code: "TOO_MANY_REQUESTS",
      requestId: expect.any(String),
    });
    expect(events.at(-1)?.rate_limited).toBe(true);
  });

  test("another key is unaffected", async () => {
    const { app } = perMinute;
    const res = await app.request("/v1/monitor", { headers: keyHeaders("b") });
    expect(res.status).toBe(200);
    const bearer = await app.request("/v1/monitor", {
      headers: { authorization: "Bearer os_oat_c" },
    });
    expect(bearer.status).toBe(200);
  });

  test("10 s bucket trips before the minute window", async () => {
    const { app } = burst;
    expect(
      (await fire(app, "/v1/monitor", 3, { headers: keyHeaders("a") })).status,
    ).toBe(200);
    expect(
      (await app.request("/v1/monitor", { headers: keyHeaders("a") })).status,
    ).toBe(429);
  });

  test("write limit covers /v1 mutations and mutating RPC methods, not reads or /mcp", async () => {
    const { app } = writes;
    const post = { method: "POST", headers: keyHeaders("a") };
    const svc = "/rpc/openstatus.monitor.v1.MonitorService";
    expect((await app.request("/v1/monitor", post)).status).toBe(200);
    expect((await app.request(`${svc}/CreateMonitor`, post)).status).toBe(200);
    const limited = await app.request(`${svc}/DeleteMonitor`, post);
    expect(limited.status).toBe(429);
    expect(limited.headers.get("retry-after")).not.toBeNull();
    // Connect clients read the code from the body, so /rpc gets the Connect shape
    expect(await limited.json()).toEqual({
      code: "resource_exhausted",
      message: "Rate limit exceeded, retry later",
    });
    // reads over POST and JSON-RPC are not writes
    expect(
      (await app.request("/v1/monitor", { headers: keyHeaders("a") })).status,
    ).toBe(200);
    expect((await app.request(`${svc}/ListMonitors`, post)).status).toBe(200);
    expect((await app.request(`${svc}/GetMonitor`, post)).status).toBe(200);
    expect(
      (await app.request(`${svc}/GetMonitor`, { headers: keyHeaders("a") }))
        .status,
    ).toBe(200);
    expect((await app.request("/mcp", post)).status).toBe(200);
  });

  test("IP fallback keys anonymous /public/status/* requests", async () => {
    const { app } = pub;
    const ip1 = { headers: { "fly-client-ip": "203.0.113.1" } };
    const ip2 = { headers: { "x-forwarded-for": "203.0.113.2, 10.0.0.1" } };
    expect((await fire(app, "/public/status/acme", 2, ip1)).status).toBe(200);
    expect((await app.request("/public/status/acme", ip1)).status).toBe(429);
    expect((await app.request("/public/status/acme", ip2)).status).toBe(200);
  });

  test("x-forwarded-for keys on the proxy-appended last entry, not the client-supplied first", async () => {
    const { app } = pub;
    // the trusted proxy appends 203.0.113.3; everything before it is attacker input
    const spoofed = (fake: string) => ({
      headers: { "x-forwarded-for": `${fake}, 203.0.113.3` },
    });
    expect(
      (await app.request("/public/status/acme", spoofed("1.1.1.1"))).status,
    ).toBe(200);
    expect(
      (await app.request("/public/status/acme", spoofed("2.2.2.2"))).status,
    ).toBe(200);
    expect(
      (await app.request("/public/status/acme", spoofed("3.3.3.3"))).status,
    ).toBe(429);
  });

  test("/ping, /openapi* and /.well-known/* are unaffected", async () => {
    const { app } = burst;
    for (const path of [
      "/ping",
      "/openapi.json",
      "/.well-known/oauth-authorization-server",
    ]) {
      expect(
        (await fire(app, path, 5, { headers: keyHeaders("a") })).status,
      ).toBe(200);
    }
  });

  test("the raw key never appears in the key generator output", async () => {
    const raw = "os_live_super_secret_key";
    const key = await (
      await keyApp.request("/k", { headers: keyHeaders(raw) })
    ).text();
    expect(key).not.toContain(raw);
    expect(key).toMatch(/^key:[0-9a-f]{16}$/);
    const bearer = await (
      await keyApp.request("/k", {
        headers: { authorization: `Bearer ${raw}` },
      })
    ).text();
    expect(bearer).toBe(key);
    const anon = await (
      await keyApp.request("/k", {
        headers: { "fly-client-ip": "198.51.100.7" },
      })
    ).text();
    expect(anon).toBe("ip:198.51.100.7");
  });

  test("a limited request never reaches the handler and Retry-After stays within the window", async () => {
    const { app, handled } = burst;
    expect(
      (await fire(app, "/v1/monitor", 3, { headers: keyHeaders("handler") }))
        .status,
    ).toBe(200);
    const before = handled();
    const limited = await app.request("/v1/monitor", {
      headers: keyHeaders("handler"),
    });
    expect(limited.status).toBe(429);
    expect(handled()).toBe(before);
    const retryAfter = Number(limited.headers.get("retry-after"));
    expect(retryAfter).toBeGreaterThanOrEqual(1);
    expect(retryAfter).toBeLessThanOrEqual(10);
    expect(await limited.json()).toMatchObject({
      message: "Rate limit exceeded, retry later",
      docs: "https://www.openstatus.dev/docs/api-references/errors/code/TOO_MANY_REQUESTS",
    });
  });

  test("the 10 s bucket recovers after its window while the minute window keeps counting", async () => {
    const { app } = recovery;
    vi.useFakeTimers();
    try {
      const init = { headers: keyHeaders("a") };
      expect((await app.request("/v1/monitor", init)).status).toBe(200);
      expect((await app.request("/v1/monitor", init)).status).toBe(429);

      vi.advanceTimersByTime(10_001);
      expect((await app.request("/v1/monitor", init)).status).toBe(200);

      // minute window (limit 2) is now full even though the burst bucket reset
      vi.advanceTimersByTime(10_001);
      expect((await app.request("/v1/monitor", init)).status).toBe(429);

      vi.advanceTimersByTime(60_001);
      expect((await app.request("/v1/monitor", init)).status).toBe(200);
    } finally {
      vi.useRealTimers();
    }
  });

  test("x-openstatus-key wins over a bearer token and fly-client-ip over x-forwarded-for", async () => {
    const { app } = precedence;
    // key + bearer counts against the key, so the same bearer alone is a different bucket
    const both = {
      headers: { "x-openstatus-key": "k1", authorization: "Bearer t1" },
    };
    expect((await app.request("/v1/monitor", both)).status).toBe(200);
    expect((await app.request("/v1/monitor", both)).status).toBe(429);
    expect(
      (
        await app.request("/v1/monitor", {
          headers: { authorization: "Bearer t1" },
        })
      ).status,
    ).toBe(200);

    const flyAndForwarded = {
      headers: {
        "fly-client-ip": "203.0.113.9",
        "x-forwarded-for": "198.51.100.9",
      },
    };
    expect((await app.request("/v1/monitor", flyAndForwarded)).status).toBe(
      200,
    );
    expect((await app.request("/v1/monitor", flyAndForwarded)).status).toBe(
      429,
    );
    expect(
      (
        await app.request("/v1/monitor", {
          headers: { "x-forwarded-for": "198.51.100.9" },
        })
      ).status,
    ).toBe(200);
  });

  test("anonymous requests without any client IP share one bucket", async () => {
    const key = await (await keyApp.request("/k")).text();
    expect(key).toBe("ip:unknown");
  });

  test("a key exhausted on /v1 is still served on /public, and vice versa", async () => {
    const { app } = isolation;
    const ip = {
      headers: { ...keyHeaders("a"), "fly-client-ip": "203.0.113.5" },
    };
    expect((await app.request("/v1/monitor", ip)).status).toBe(200);
    expect((await app.request("/v1/monitor", ip)).status).toBe(429);
    expect((await app.request("/public/status/acme", ip)).status).toBe(200);
    expect((await app.request("/public/status/acme", ip)).status).toBe(429);
    // the public bucket being full does not touch a fresh key on /v1
    expect(
      (
        await app.request("/v1/monitor", {
          headers: {
            ...keyHeaders("b"),
            ...ip.headers,
            "x-openstatus-key": "b",
          },
        })
      ).status,
    ).toBe(200);
  });

  test("unauthenticated OAuth registration and token POSTs count as writes per IP", async () => {
    const { app } = writes;
    const post = (path: string) =>
      app.request(path, {
        method: "POST",
        headers: { "fly-client-ip": "203.0.113.42" },
      });
    expect((await post("/oauth/register")).status).toBe(200);
    expect((await post("/oauth/token")).status).toBe(200);
    expect((await post("/oauth/revoke")).status).toBe(429);
    // authorize is a redirect, not a write
    expect(
      (
        await app.request("/oauth/authorize", {
          headers: { "fly-client-ip": "203.0.113.42" },
        })
      ).status,
    ).toBe(200);
  });

  test("rotating bad credentials from one IP is capped by the 401 budget, successes are refunded", async () => {
    const { app } = authFailures;
    const from = (ip: string, key: string) =>
      app.request("/v1/monitor", {
        headers: { ...keyHeaders(key), "fly-client-ip": ip },
      });
    // valid traffic never counts, however much of it there is
    for (let i = 0; i < 5; i++) {
      expect((await from("203.0.113.10", `valid-${i}`)).status).toBe(200);
    }
    expect((await from("203.0.113.10", "fake-1")).status).toBe(401);
    expect((await from("203.0.113.10", "fake-2")).status).toBe(401);
    expect((await from("203.0.113.10", "fake-3")).status).toBe(429);
    // another IP has its own budget; anonymous requests are not credential rotation
    expect((await from("203.0.113.11", "fake-4")).status).toBe(401);
    expect(
      (
        await app.request("/v1/monitor", {
          headers: { "fly-client-ip": "203.0.113.10" },
        })
      ).status,
    ).toBe(401);
  });
});