Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
import { type Context, Hono } from "hono";
import { requestId } from "hono/request-id";
import { describe, expect, test, vi } from "vitest";
import { createRateLimit, credentialKey } from "./rate-limit";
type Env = { Variables: { event: Record<string, unknown> } };
const HUGE = 1_000_000;
// Limiters own an interval timer; build them at module scope, outside the sanitizer.
function build(
config: {
perMinute?: number;
burstPer10s?: number;
writesPerMinute?: number;
publicPerMinute?: number;
},
handler: (c: Context<Env>) => Response = (c) => c.text("ok"),
) {
const app = new Hono<Env>();
const events: Record<string, unknown>[] = [];
let handled = 0;
app.use("*", requestId());
app.use("*", async (c, next) => {
const event: Record<string, unknown> = {};
events.push(event);
c.set("event", event);
await next();
});
app.use(
"*",
...createRateLimit({
perMinute: HUGE,
burstPer10s: HUGE,
writesPerMinute: HUGE,
publicPerMinute: HUGE,
...config,
}),
);
app.all("*", (c) => {
handled++;
return handler(c);
});
return { app, events, handled: () => handled };
}
const perMinute = build({ perMinute: 600 });
const burst = build({ burstPer10s: 3 });
const writes = build({ writesPerMinute: 2 });
const pub = build({ publicPerMinute: 2 });
const recovery = build({ burstPer10s: 1, perMinute: 2 });
const precedence = build({ burstPer10s: 1 });
const isolation = build({ perMinute: 1, publicPerMinute: 1 });
// stands in for the auth middleware: `valid*` credentials are accepted
const authFailures = build({ perMinute: 2 }, (c) =>
c.req.header("x-openstatus-key")?.startsWith("valid")
? c.text("ok")
: c.text("nope", 401),
);
const keyApp = new Hono().get("/k", async (c) =>
c.text(await credentialKey(c)),
);
const keyHeaders = (key: string) => ({ "x-openstatus-key": key });
async function fire(
app: Hono<Env>,
path: string,
n: number,
init?: RequestInit,
) {
let last: Response | undefined;
for (let i = 0; i < n; i++) last = await app.request(path, init);
return last as Response;
}
describe("rate limit", () => {
test("601st request in a window from one key gets 429", async () => {
const { app, events } = perMinute;
const ok = await fire(app, "/v1/monitor", 600, {
headers: keyHeaders("a"),
});
expect(ok.status).toBe(200);
const limited = await app.request("/v1/monitor", {
headers: keyHeaders("a"),
});
expect(limited.status).toBe(429);
expect(Number(limited.headers.get("retry-after"))).toBeGreaterThan(0);
expect(await limited.json()).toMatchObject({
code: "TOO_MANY_REQUESTS",
requestId: expect.any(String),
});
expect(events.at(-1)?.rate_limited).toBe(true);
});
test("another key is unaffected", async () => {
const { app } = perMinute;
const res = await app.request("/v1/monitor", { headers: keyHeaders("b") });
expect(res.status).toBe(200);
const bearer = await app.request("/v1/monitor", {
headers: { authorization: "Bearer os_oat_c" },
});
expect(bearer.status).toBe(200);
});
test("10 s bucket trips before the minute window", async () => {
const { app } = burst;
expect(
(await fire(app, "/v1/monitor", 3, { headers: keyHeaders("a") })).status,
).toBe(200);
expect(
(await app.request("/v1/monitor", { headers: keyHeaders("a") })).status,
).toBe(429);
});
test("write limit covers /v1 mutations and mutating RPC methods, not reads or /mcp", async () => {
const { app } = writes;
const post = { method: "POST", headers: keyHeaders("a") };
const svc = "/rpc/openstatus.monitor.v1.MonitorService";
expect((await app.request("/v1/monitor", post)).status).toBe(200);
expect((await app.request(`${svc}/CreateMonitor`, post)).status).toBe(200);
const limited = await app.request(`${svc}/DeleteMonitor`, post);
expect(limited.status).toBe(429);
expect(limited.headers.get("retry-after")).not.toBeNull();
// Connect clients read the code from the body, so /rpc gets the Connect shape
expect(await limited.json()).toEqual({
code: "resource_exhausted",
message: "Rate limit exceeded, retry later",
});
// reads over POST and JSON-RPC are not writes
expect(
(await app.request("/v1/monitor", { headers: keyHeaders("a") })).status,
).toBe(200);
expect((await app.request(`${svc}/ListMonitors`, post)).status).toBe(200);
expect((await app.request(`${svc}/GetMonitor`, post)).status).toBe(200);
expect(
(await app.request(`${svc}/GetMonitor`, { headers: keyHeaders("a") }))
.status,
).toBe(200);
expect((await app.request("/mcp", post)).status).toBe(200);
});
test("IP fallback keys anonymous /public/status/* requests", async () => {
const { app } = pub;
const ip1 = { headers: { "fly-client-ip": "203.0.113.1" } };
const ip2 = { headers: { "x-forwarded-for": "203.0.113.2, 10.0.0.1" } };
expect((await fire(app, "/public/status/acme", 2, ip1)).status).toBe(200);
expect((await app.request("/public/status/acme", ip1)).status).toBe(429);
expect((await app.request("/public/status/acme", ip2)).status).toBe(200);
});
test("x-forwarded-for keys on the proxy-appended last entry, not the client-supplied first", async () => {
const { app } = pub;
// the trusted proxy appends 203.0.113.3; everything before it is attacker input
const spoofed = (fake: string) => ({
headers: { "x-forwarded-for": `${fake}, 203.0.113.3` },
});
expect(
(await app.request("/public/status/acme", spoofed("1.1.1.1"))).status,
).toBe(200);
expect(
(await app.request("/public/status/acme", spoofed("2.2.2.2"))).status,
).toBe(200);
expect(
(await app.request("/public/status/acme", spoofed("3.3.3.3"))).status,
).toBe(429);
});
test("/ping, /openapi* and /.well-known/* are unaffected", async () => {
const { app } = burst;
for (const path of [
"/ping",
"/openapi.json",
"/.well-known/oauth-authorization-server",
]) {
expect(
(await fire(app, path, 5, { headers: keyHeaders("a") })).status,
).toBe(200);
}
});
test("the raw key never appears in the key generator output", async () => {
const raw = "os_live_super_secret_key";
const key = await (
await keyApp.request("/k", { headers: keyHeaders(raw) })
).text();
expect(key).not.toContain(raw);
expect(key).toMatch(/^key:[0-9a-f]{16}$/);
const bearer = await (
await keyApp.request("/k", {
headers: { authorization: `Bearer ${raw}` },
})
).text();
expect(bearer).toBe(key);
const anon = await (
await keyApp.request("/k", {
headers: { "fly-client-ip": "198.51.100.7" },
})
).text();
expect(anon).toBe("ip:198.51.100.7");
});
test("a limited request never reaches the handler and Retry-After stays within the window", async () => {
const { app, handled } = burst;
expect(
(await fire(app, "/v1/monitor", 3, { headers: keyHeaders("handler") }))
.status,
).toBe(200);
const before = handled();
const limited = await app.request("/v1/monitor", {
headers: keyHeaders("handler"),
});
expect(limited.status).toBe(429);
expect(handled()).toBe(before);
const retryAfter = Number(limited.headers.get("retry-after"));
expect(retryAfter).toBeGreaterThanOrEqual(1);
expect(retryAfter).toBeLessThanOrEqual(10);
expect(await limited.json()).toMatchObject({
message: "Rate limit exceeded, retry later",
docs: "https://www.openstatus.dev/docs/api-references/errors/code/TOO_MANY_REQUESTS",
});
});
test("the 10 s bucket recovers after its window while the minute window keeps counting", async () => {
const { app } = recovery;
vi.useFakeTimers();
try {
const init = { headers: keyHeaders("a") };
expect((await app.request("/v1/monitor", init)).status).toBe(200);
expect((await app.request("/v1/monitor", init)).status).toBe(429);
vi.advanceTimersByTime(10_001);
expect((await app.request("/v1/monitor", init)).status).toBe(200);
// minute window (limit 2) is now full even though the burst bucket reset
vi.advanceTimersByTime(10_001);
expect((await app.request("/v1/monitor", init)).status).toBe(429);
vi.advanceTimersByTime(60_001);
expect((await app.request("/v1/monitor", init)).status).toBe(200);
} finally {
vi.useRealTimers();
}
});
test("x-openstatus-key wins over a bearer token and fly-client-ip over x-forwarded-for", async () => {
const { app } = precedence;
// key + bearer counts against the key, so the same bearer alone is a different bucket
const both = {
headers: { "x-openstatus-key": "k1", authorization: "Bearer t1" },
};
expect((await app.request("/v1/monitor", both)).status).toBe(200);
expect((await app.request("/v1/monitor", both)).status).toBe(429);
expect(
(
await app.request("/v1/monitor", {
headers: { authorization: "Bearer t1" },
})
).status,
).toBe(200);
const flyAndForwarded = {
headers: {
"fly-client-ip": "203.0.113.9",
"x-forwarded-for": "198.51.100.9",
},
};
expect((await app.request("/v1/monitor", flyAndForwarded)).status).toBe(
200,
);
expect((await app.request("/v1/monitor", flyAndForwarded)).status).toBe(
429,
);
expect(
(
await app.request("/v1/monitor", {
headers: { "x-forwarded-for": "198.51.100.9" },
})
).status,
).toBe(200);
});
test("anonymous requests without any client IP share one bucket", async () => {
const key = await (await keyApp.request("/k")).text();
expect(key).toBe("ip:unknown");
});
test("a key exhausted on /v1 is still served on /public, and vice versa", async () => {
const { app } = isolation;
const ip = {
headers: { ...keyHeaders("a"), "fly-client-ip": "203.0.113.5" },
};
expect((await app.request("/v1/monitor", ip)).status).toBe(200);
expect((await app.request("/v1/monitor", ip)).status).toBe(429);
expect((await app.request("/public/status/acme", ip)).status).toBe(200);
expect((await app.request("/public/status/acme", ip)).status).toBe(429);
// the public bucket being full does not touch a fresh key on /v1
expect(
(
await app.request("/v1/monitor", {
headers: {
...keyHeaders("b"),
...ip.headers,
"x-openstatus-key": "b",
},
})
).status,
).toBe(200);
});
test("unauthenticated OAuth registration and token POSTs count as writes per IP", async () => {
const { app } = writes;
const post = (path: string) =>
app.request(path, {
method: "POST",
headers: { "fly-client-ip": "203.0.113.42" },
});
expect((await post("/oauth/register")).status).toBe(200);
expect((await post("/oauth/token")).status).toBe(200);
expect((await post("/oauth/revoke")).status).toBe(429);
// authorize is a redirect, not a write
expect(
(
await app.request("/oauth/authorize", {
headers: { "fly-client-ip": "203.0.113.42" },
})
).status,
).toBe(200);
});
test("rotating bad credentials from one IP is capped by the 401 budget, successes are refunded", async () => {
const { app } = authFailures;
const from = (ip: string, key: string) =>
app.request("/v1/monitor", {
headers: { ...keyHeaders(key), "fly-client-ip": ip },
});
// valid traffic never counts, however much of it there is
for (let i = 0; i < 5; i++) {
expect((await from("203.0.113.10", `valid-${i}`)).status).toBe(200);
}
expect((await from("203.0.113.10", "fake-1")).status).toBe(401);
expect((await from("203.0.113.10", "fake-2")).status).toBe(401);
expect((await from("203.0.113.10", "fake-3")).status).toBe(429);
// another IP has its own budget; anonymous requests are not credential rotation
expect((await from("203.0.113.11", "fake-4")).status).toBe(401);
expect(
(
await app.request("/v1/monitor", {
headers: { "fly-client-ip": "203.0.113.10" },
})
).status,
).toBe(401);
});
});