Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
fix: cache api key auth lookup to survive transient turso 5xx (#2257)
Colin Ozannef6ec391ab6f1
diff --git a/apps/server/src/libs/middlewares/auth.ts b/apps/server/src/libs/middlewares/auth.tsindex 7d38448ba..c4c5782aa 100644--- a/apps/server/src/libs/middlewares/auth.ts+++ b/apps/server/src/libs/middlewares/auth.ts@@ -10,6 +10,11 @@ import {shouldUpdateLastUsed,verifyApiKeyHash,} from "@openstatus/db/src/utils/api-key";+import {+ isRetryableDbError,+ isTransientServerError,+ withBusyRetry,+} from "@openstatus/services";import { UnkeyCore } from "@unkey/api/core";import { keysVerifyKey } from "@unkey/api/funcs/keysVerifyKey";import type { Context, Next } from "hono";@@ -20,16 +25,14 @@ import type { Variables } from "@/types";const logger = getLogger("api-server");-/**- * Looks up a workspace by ID and validates the data.- * Throws OpenStatusApiError if workspace is not found or invalid.- */+// Reads only: a 502 can land after a write partially applied.+const retryRead = <T>(fn: () => Promise<T>): Promise<T> =>+ withBusyRetry(fn, (e) => isRetryableDbError(e) || isTransientServerError(e));+export async function lookupWorkspace(workspaceId: number) {- const _workspace = await db- .select()- .from(workspace)- .where(eq(workspace.id, workspaceId))- .get();+ const _workspace = await retryRead(() =>+ db.select().from(workspace).where(eq(workspace.id, workspaceId)).get(),+ );if (!_workspace) {throw new OpenStatusApiError({@@ -159,21 +162,17 @@ export async function validateKey(key: string): Promise<{if (key.startsWith("os_")) {// 1. Try custom DB firstconst prefix = key.slice(0, 11); // "os_" (3 chars) + 8 hex chars = 11 total- const customKey = await db- .select()- .from(apiKey)- .where(eq(apiKey.prefix, prefix))- .get();+ const customKey = await retryRead(() =>+ db.select().from(apiKey).where(eq(apiKey.prefix, prefix)).get(),+ );if (customKey) {- // Verify hash using bcrypt-compatible verificationif (!(await verifyApiKeyHash(key, customKey.hashedToken))) {return {result: { valid: false },error: { message: "Invalid API Key" },};}- // Check expirationif (customKey.expiresAt && customKey.expiresAt < new Date()) {return {result: { valid: false },@@ -181,12 +180,13 @@ export async function validateKey(key: string): Promise<{};}- // Update lastUsedAt (debounced)+ // Best-effort: a failed bookkeeping write must never fail auth.if (shouldUpdateLastUsed(customKey.lastUsedAt)) {await db.update(apiKey).set({ lastUsedAt: new Date() })- .where(eq(apiKey.id, customKey.id));+ .where(eq(apiKey.id, customKey.id))+ .catch(() => {});}return {result: {diff --git a/apps/server/src/routes/rpc/handlers/status-page/__tests__/status-page.test.ts b/apps/server/src/routes/rpc/handlers/status-page/__tests__/status-page.test.tsindex 14b3950e1..6daf779fe 100644--- a/apps/server/src/routes/rpc/handlers/status-page/__tests__/status-page.test.ts+++ b/apps/server/src/routes/rpc/handlers/status-page/__tests__/status-page.test.ts@@ -2214,26 +2214,31 @@ describe("StatusPageService.CreatePageSubscription", () => {});test("rejects when workspace has status-subscribers disabled", async () => {- // Temporarily disable the plan flag, hit the endpoint, then restore.- await db.run(- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."status-subscribers"', json('false')) WHERE id = 1`,- );+ const freePage = await db+ .insert(page)+ .values({+ workspaceId: 2,+ title: `${TEST_PREFIX}-plan-gate`,+ slug: `${TEST_PREFIX}-plan-gate`,+ description: "Test page for plan gate",+ customDomain: "",+ })+ .returning()+ .get();try {const res = await connectRequest("CreatePageSubscription",{- pageId: String(testPageId),+ pageId: String(freePage.id),emailChannel: {email: `${TEST_PREFIX}[email protected]`,},},- { "x-openstatus-key": "1" },+ { "x-openstatus-key": "2" },);expect(res.status).toBe(403);} finally {- await db.run(- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."status-subscribers"', json('true')) WHERE id = 1`,- );+ await db.delete(page).where(eq(page.id, freePage.id));}});});@@ -3501,29 +3506,32 @@ describe("StatusPageService — allow_index", () => {});test("rejects allow_index=false when plan does not support it", async () => {- // Temporarily disable no-index feature on workspace- await db.run(- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."no-index"', json('false')) WHERE id = 1`,- );-+ const freePage = await db+ .insert(page)+ .values({+ workspaceId: 2,+ title: `${TEST_PREFIX}-no-index`,+ slug: `${TEST_PREFIX}-no-index`,+ description: "Test page for no-index gate",+ customDomain: "",+ })+ .returning()+ .get();try {const res = await connectRequest("UpdateStatusPage",{- id: String(allowIndexPageId),+ id: String(freePage.id),allowIndex: false,},- { "x-openstatus-key": "1" },+ { "x-openstatus-key": "2" },);expect(res.status).not.toBe(200);const data = await res.json();- expect(data.message).toContain("Upgrade");+ expect(data.message).toContain("Upgrade for search engine indexing");} finally {- // Re-enable no-index feature- await db.run(- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."no-index"', json('true')) WHERE id = 1`,- );+ await db.delete(page).where(eq(page.id, freePage.id));}});diff --git a/apps/server/src/routes/slack/handler.test.ts b/apps/server/src/routes/slack/handler.test.tsindex 5d18ce21e..86e01f694 100644--- a/apps/server/src/routes/slack/handler.test.ts+++ b/apps/server/src/routes/slack/handler.test.ts@@ -3,6 +3,8 @@ import crypto from "node:crypto";import { Hono } from "hono";+import * as agentModule from "./agent";+const SIGNING_SECRET = "test-signing-secret";process.env.SLACK_SIGNING_SECRET = SIGNING_SECRET;@@ -64,6 +66,7 @@ let runAgentOverride:| null = null;mock.module("./agent", () => ({+ ...agentModule,runAgent: () => {if (runAgentOverride) return runAgentOverride();return Promise.resolve({diff --git a/packages/services/src/__tests__/retry.test.ts b/packages/services/src/__tests__/retry.test.tsindex e524956b3..b2a812e93 100644--- a/packages/services/src/__tests__/retry.test.ts+++ b/packages/services/src/__tests__/retry.test.ts@@ -1,7 +1,11 @@import { describe, expect, test } from "bun:test";import { NotFoundError } from "../errors";-import { isRetryableDbError, withBusyRetry } from "../retry";+import {+ isRetryableDbError,+ isTransientServerError,+ withBusyRetry,+} from "../retry";describe("isRetryableDbError", () => {test("true for SQLITE_BUSY code", () => {@@ -38,6 +42,48 @@ describe("isRetryableDbError", () => {});});+describe("isTransientServerError", () => {+ test("true for a 5xx status (HttpServerError cause)", () => {+ expect(isTransientServerError({ code: "SERVER_ERROR", status: 502 })).toBe(+ true,+ );+ });++ test("true for a 5xx message without a status", () => {+ expect(+ isTransientServerError(new Error("Server returned HTTP status 503")),+ ).toBe(true);+ });++ test("true for the drizzle-wrapped 5xx cause chain", () => {+ const err = new Error("Failed query: select ...");+ (err as Error & { cause: unknown }).cause = {+ code: "SERVER_ERROR",+ status: 502,+ message: "SERVER_ERROR: Server returned HTTP status 502",+ };+ expect(isTransientServerError(err)).toBe(true);+ });++ test("false for a non-transient 4xx SERVER_ERROR", () => {+ expect(+ isTransientServerError({+ code: "SERVER_ERROR",+ status: 404,+ message: "Server returned HTTP status 404",+ }),+ ).toBe(false);+ });++ test("false for a bare SERVER_ERROR with no 5xx signal", () => {+ expect(isTransientServerError({ code: "SERVER_ERROR" })).toBe(false);+ });++ test("false for a busy/locked error (handled by isRetryableDbError)", () => {+ expect(isTransientServerError({ code: "SQLITE_BUSY" })).toBe(false);+ });+});+describe("withBusyRetry", () => {test("resolves once a transient busy error clears", async () => {let attempts = 0;@@ -71,4 +117,29 @@ describe("withBusyRetry", () => {await expect(promise).rejects.toBe(original);expect(attempts).toBe(1);});++ test("honours a custom predicate (e.g. transient 5xx for reads)", async () => {+ let attempts = 0;+ const result = await withBusyRetry(+ async () => {+ attempts++;+ if (attempts < 2) throw { code: "SERVER_ERROR", status: 503 };+ return "ok";+ },+ (e) => isRetryableDbError(e) || isTransientServerError(e),+ );+ expect(result).toBe("ok");+ expect(attempts).toBe(2);+ });++ test("default predicate does not retry a transient 5xx", async () => {+ let attempts = 0;+ const original = { code: "SERVER_ERROR", status: 503 };+ const promise = withBusyRetry(async () => {+ attempts++;+ throw original;+ });+ await expect(promise).rejects.toBe(original);+ expect(attempts).toBe(1);+ });});diff --git a/packages/services/src/index.ts b/packages/services/src/index.tsindex 61561d002..f68adba09 100644--- a/packages/services/src/index.ts+++ b/packages/services/src/index.ts@@ -11,7 +11,11 @@ export {withTransaction,} from "./context";-export { isRetryableDbError, withBusyRetry } from "./retry";+export {+ isRetryableDbError,+ isTransientServerError,+ withBusyRetry,+} from "./retry";export {ConflictError,diff --git a/packages/services/src/page-subscriber/__tests__/page-subscriber.test.ts b/packages/services/src/page-subscriber/__tests__/page-subscriber.test.tsindex 903fb6d42..abbfd66c4 100644--- a/packages/services/src/page-subscriber/__tests__/page-subscriber.test.ts+++ b/packages/services/src/page-subscriber/__tests__/page-subscriber.test.ts@@ -8,9 +8,10 @@ import {test,} from "bun:test";-import { and, db, eq, sql } from "@openstatus/db";+import { and, db, eq } from "@openstatus/db";import {auditLog,+ page,pageSubscriber,pageSubscriberToPageComponent,} from "@openstatus/db/src/schema";@@ -24,7 +25,10 @@ import {upsertSelfSignupSubscriber,verifySelfSignupSubscriber,} from "..";-import { SEEDED_WORKSPACE_TEAM_ID } from "../../../test/fixtures";+import {+ SEEDED_WORKSPACE_FREE_ID,+ SEEDED_WORKSPACE_TEAM_ID,+} from "../../../test/fixtures";import {clearAuditLog,expectAuditRow,@@ -286,23 +290,26 @@ describe("upsertSelfSignupSubscriber", () => {).rejects.toThrow();});- // Plan-gate: temporarily flip workspace 1's `status-subscribers` to- // false, hit the function, then restore. Mirrors the pattern in- // `apps/server/.../status-page.test.ts`.test("rejects when the workspace plan disables status-subscribers", async () => {- await db.run(- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."status-subscribers"', json('false')) WHERE id = ${SEEDED_WORKSPACE_TEAM_ID}`,- );+ const freePage = await db+ .insert(page)+ .values({+ workspaceId: SEEDED_WORKSPACE_FREE_ID,+ title: "plan-gate",+ description: "plan-gate",+ slug: `plan-gate-${Date.now()}`,+ customDomain: "",+ })+ .returning()+ .get();try {await expect(upsertSelfSignupSubscriber({- input: { email: EMAILS.planGate, pageId: PAGE_ID },+ input: { email: EMAILS.planGate, pageId: freePage.id },}),).rejects.toThrow("Upgrade to use status subscribers");} finally {- await db.run(- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."status-subscribers"', json('true')) WHERE id = ${SEEDED_WORKSPACE_TEAM_ID}`,- );+ await db.delete(page).where(eq(page.id, freePage.id));}});});diff --git a/packages/services/src/retry.ts b/packages/services/src/retry.tsindex 73985a057..98bd7dee8 100644--- a/packages/services/src/retry.ts+++ b/packages/services/src/retry.ts@@ -1,5 +1,6 @@const RETRYABLE_CODES = new Set(["SQLITE_BUSY", "SQLITE_LOCKED"]);const RETRYABLE_MESSAGE = /database is (locked|busy)/i;+const TRANSIENT_SERVER_MESSAGE = /Server returned HTTP status 5\d\d/i;const MAX_ATTEMPTS = 5;const BASE_DELAY_MS = 25;const MAX_DELAY_MS = 400;@@ -17,6 +18,10 @@ function hasCause(value: object): value is { cause: unknown } {return "cause" in value;}+function hasStatus(value: object): value is { status: unknown } {+ return "status" in value;+}+export function isRetryableDbError(err: unknown): boolean {let current: unknown = err;for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {@@ -34,6 +39,28 @@ export function isRetryableDbError(err: unknown): boolean {return false;}+// Transient libSQL/Turso 5xx. Safe to retry only for idempotent reads —+// a 502 may land after a write partially applied. libsql maps every+// HttpServerError to code "SERVER_ERROR" regardless of status, so gate on the+// 5xx status (carried on the HttpServerError cause) or the 5xx message — never+// the bare code, which also covers non-retryable 4xx (bad token, db not found).+export function isTransientServerError(err: unknown): boolean {+ let current: unknown = err;+ for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {+ if (typeof current !== "object" || current === null) return false;++ if (hasStatus(current) && typeof current.status === "number") {+ if (current.status >= 500 && current.status <= 599) return true;+ }+ if (hasMessage(current) && typeof current.message === "string") {+ if (TRANSIENT_SERVER_MESSAGE.test(current.message)) return true;+ }+ if (!hasCause(current) || current.cause === current) return false;+ current = current.cause;+ }+ return false;+}+function backoffDelayMs(attempt: number): number {const cap = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** attempt);return Math.random() * cap;@@ -41,12 +68,15 @@ function backoffDelayMs(attempt: number): number {const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));-export async function withBusyRetry<T>(fn: () => Promise<T>): Promise<T> {+export async function withBusyRetry<T>(+ fn: () => Promise<T>,+ isRetryable: (err: unknown) => boolean = isRetryableDbError,+): Promise<T> {for (let attempt = 0; ; attempt++) {try {return await fn();} catch (err) {- if (attempt >= MAX_ATTEMPTS - 1 || !isRetryableDbError(err)) throw err;+ if (attempt >= MAX_ATTEMPTS - 1 || !isRetryable(err)) throw err;await sleep(backoffDelayMs(attempt));}}