Skip to content

Open-source status page, incident response and uptime monitoring

fix: cache api key auth lookup to survive transient turso 5xx (#2257)

Colin Ozannef6ec391ab6f1
diff --git a/apps/server/src/libs/middlewares/auth.ts b/apps/server/src/libs/middlewares/auth.ts
index 7d38448ba..c4c5782aa 100644
--- a/apps/server/src/libs/middlewares/auth.ts
+++ b/apps/server/src/libs/middlewares/auth.ts
@@ -10,6 +10,11 @@ import {
shouldUpdateLastUsed,
verifyApiKeyHash,
} from "@openstatus/db/src/utils/api-key";
+import {
+ isRetryableDbError,
+ isTransientServerError,
+ withBusyRetry,
+} from "@openstatus/services";
import { UnkeyCore } from "@unkey/api/core";
import { keysVerifyKey } from "@unkey/api/funcs/keysVerifyKey";
import type { Context, Next } from "hono";
@@ -20,16 +25,14 @@ import type { Variables } from "@/types";
const logger = getLogger("api-server");
-/**
- * Looks up a workspace by ID and validates the data.
- * Throws OpenStatusApiError if workspace is not found or invalid.
- */
+// Reads only: a 502 can land after a write partially applied.
+const retryRead = <T>(fn: () => Promise<T>): Promise<T> =>
+ withBusyRetry(fn, (e) => isRetryableDbError(e) || isTransientServerError(e));
+
export async function lookupWorkspace(workspaceId: number) {
- const _workspace = await db
- .select()
- .from(workspace)
- .where(eq(workspace.id, workspaceId))
- .get();
+ const _workspace = await retryRead(() =>
+ db.select().from(workspace).where(eq(workspace.id, workspaceId)).get(),
+ );
if (!_workspace) {
throw new OpenStatusApiError({
@@ -159,21 +162,17 @@ export async function validateKey(key: string): Promise<{
if (key.startsWith("os_")) {
// 1. Try custom DB first
const prefix = key.slice(0, 11); // "os_" (3 chars) + 8 hex chars = 11 total
- const customKey = await db
- .select()
- .from(apiKey)
- .where(eq(apiKey.prefix, prefix))
- .get();
+ const customKey = await retryRead(() =>
+ db.select().from(apiKey).where(eq(apiKey.prefix, prefix)).get(),
+ );
if (customKey) {
- // Verify hash using bcrypt-compatible verification
if (!(await verifyApiKeyHash(key, customKey.hashedToken))) {
return {
result: { valid: false },
error: { message: "Invalid API Key" },
};
}
- // Check expiration
if (customKey.expiresAt && customKey.expiresAt < new Date()) {
return {
result: { valid: false },
@@ -181,12 +180,13 @@ export async function validateKey(key: string): Promise<{
};
}
- // Update lastUsedAt (debounced)
+ // Best-effort: a failed bookkeeping write must never fail auth.
if (shouldUpdateLastUsed(customKey.lastUsedAt)) {
await db
.update(apiKey)
.set({ lastUsedAt: new Date() })
- .where(eq(apiKey.id, customKey.id));
+ .where(eq(apiKey.id, customKey.id))
+ .catch(() => {});
}
return {
result: {
diff --git a/apps/server/src/routes/rpc/handlers/status-page/__tests__/status-page.test.ts b/apps/server/src/routes/rpc/handlers/status-page/__tests__/status-page.test.ts
index 14b3950e1..6daf779fe 100644
--- a/apps/server/src/routes/rpc/handlers/status-page/__tests__/status-page.test.ts
+++ b/apps/server/src/routes/rpc/handlers/status-page/__tests__/status-page.test.ts
@@ -2214,26 +2214,31 @@ describe("StatusPageService.CreatePageSubscription", () => {
});
test("rejects when workspace has status-subscribers disabled", async () => {
- // Temporarily disable the plan flag, hit the endpoint, then restore.
- await db.run(
- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."status-subscribers"', json('false')) WHERE id = 1`,
- );
+ const freePage = await db
+ .insert(page)
+ .values({
+ workspaceId: 2,
+ title: `${TEST_PREFIX}-plan-gate`,
+ slug: `${TEST_PREFIX}-plan-gate`,
+ description: "Test page for plan gate",
+ customDomain: "",
+ })
+ .returning()
+ .get();
try {
const res = await connectRequest(
"CreatePageSubscription",
{
- pageId: String(testPageId),
+ pageId: String(freePage.id),
emailChannel: {
email: `${TEST_PREFIX}[email protected]`,
},
},
- { "x-openstatus-key": "1" },
+ { "x-openstatus-key": "2" },
);
expect(res.status).toBe(403);
} finally {
- await db.run(
- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."status-subscribers"', json('true')) WHERE id = 1`,
- );
+ await db.delete(page).where(eq(page.id, freePage.id));
}
});
});
@@ -3501,29 +3506,32 @@ describe("StatusPageService — allow_index", () => {
});
test("rejects allow_index=false when plan does not support it", async () => {
- // Temporarily disable no-index feature on workspace
- await db.run(
- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."no-index"', json('false')) WHERE id = 1`,
- );
-
+ const freePage = await db
+ .insert(page)
+ .values({
+ workspaceId: 2,
+ title: `${TEST_PREFIX}-no-index`,
+ slug: `${TEST_PREFIX}-no-index`,
+ description: "Test page for no-index gate",
+ customDomain: "",
+ })
+ .returning()
+ .get();
try {
const res = await connectRequest(
"UpdateStatusPage",
{
- id: String(allowIndexPageId),
+ id: String(freePage.id),
allowIndex: false,
},
- { "x-openstatus-key": "1" },
+ { "x-openstatus-key": "2" },
);
expect(res.status).not.toBe(200);
const data = await res.json();
- expect(data.message).toContain("Upgrade");
+ expect(data.message).toContain("Upgrade for search engine indexing");
} finally {
- // Re-enable no-index feature
- await db.run(
- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."no-index"', json('true')) WHERE id = 1`,
- );
+ await db.delete(page).where(eq(page.id, freePage.id));
}
});
diff --git a/apps/server/src/routes/slack/handler.test.ts b/apps/server/src/routes/slack/handler.test.ts
index 5d18ce21e..86e01f694 100644
--- a/apps/server/src/routes/slack/handler.test.ts
+++ b/apps/server/src/routes/slack/handler.test.ts
@@ -3,6 +3,8 @@ import crypto from "node:crypto";
import { Hono } from "hono";
+import * as agentModule from "./agent";
+
const SIGNING_SECRET = "test-signing-secret";
process.env.SLACK_SIGNING_SECRET = SIGNING_SECRET;
@@ -64,6 +66,7 @@ let runAgentOverride:
| null = null;
mock.module("./agent", () => ({
+ ...agentModule,
runAgent: () => {
if (runAgentOverride) return runAgentOverride();
return Promise.resolve({
diff --git a/packages/services/src/__tests__/retry.test.ts b/packages/services/src/__tests__/retry.test.ts
index e524956b3..b2a812e93 100644
--- a/packages/services/src/__tests__/retry.test.ts
+++ b/packages/services/src/__tests__/retry.test.ts
@@ -1,7 +1,11 @@
import { describe, expect, test } from "bun:test";
import { NotFoundError } from "../errors";
-import { isRetryableDbError, withBusyRetry } from "../retry";
+import {
+ isRetryableDbError,
+ isTransientServerError,
+ withBusyRetry,
+} from "../retry";
describe("isRetryableDbError", () => {
test("true for SQLITE_BUSY code", () => {
@@ -38,6 +42,48 @@ describe("isRetryableDbError", () => {
});
});
+describe("isTransientServerError", () => {
+ test("true for a 5xx status (HttpServerError cause)", () => {
+ expect(isTransientServerError({ code: "SERVER_ERROR", status: 502 })).toBe(
+ true,
+ );
+ });
+
+ test("true for a 5xx message without a status", () => {
+ expect(
+ isTransientServerError(new Error("Server returned HTTP status 503")),
+ ).toBe(true);
+ });
+
+ test("true for the drizzle-wrapped 5xx cause chain", () => {
+ const err = new Error("Failed query: select ...");
+ (err as Error & { cause: unknown }).cause = {
+ code: "SERVER_ERROR",
+ status: 502,
+ message: "SERVER_ERROR: Server returned HTTP status 502",
+ };
+ expect(isTransientServerError(err)).toBe(true);
+ });
+
+ test("false for a non-transient 4xx SERVER_ERROR", () => {
+ expect(
+ isTransientServerError({
+ code: "SERVER_ERROR",
+ status: 404,
+ message: "Server returned HTTP status 404",
+ }),
+ ).toBe(false);
+ });
+
+ test("false for a bare SERVER_ERROR with no 5xx signal", () => {
+ expect(isTransientServerError({ code: "SERVER_ERROR" })).toBe(false);
+ });
+
+ test("false for a busy/locked error (handled by isRetryableDbError)", () => {
+ expect(isTransientServerError({ code: "SQLITE_BUSY" })).toBe(false);
+ });
+});
+
describe("withBusyRetry", () => {
test("resolves once a transient busy error clears", async () => {
let attempts = 0;
@@ -71,4 +117,29 @@ describe("withBusyRetry", () => {
await expect(promise).rejects.toBe(original);
expect(attempts).toBe(1);
});
+
+ test("honours a custom predicate (e.g. transient 5xx for reads)", async () => {
+ let attempts = 0;
+ const result = await withBusyRetry(
+ async () => {
+ attempts++;
+ if (attempts < 2) throw { code: "SERVER_ERROR", status: 503 };
+ return "ok";
+ },
+ (e) => isRetryableDbError(e) || isTransientServerError(e),
+ );
+ expect(result).toBe("ok");
+ expect(attempts).toBe(2);
+ });
+
+ test("default predicate does not retry a transient 5xx", async () => {
+ let attempts = 0;
+ const original = { code: "SERVER_ERROR", status: 503 };
+ const promise = withBusyRetry(async () => {
+ attempts++;
+ throw original;
+ });
+ await expect(promise).rejects.toBe(original);
+ expect(attempts).toBe(1);
+ });
});
diff --git a/packages/services/src/index.ts b/packages/services/src/index.ts
index 61561d002..f68adba09 100644
--- a/packages/services/src/index.ts
+++ b/packages/services/src/index.ts
@@ -11,7 +11,11 @@ export {
withTransaction,
} from "./context";
-export { isRetryableDbError, withBusyRetry } from "./retry";
+export {
+ isRetryableDbError,
+ isTransientServerError,
+ withBusyRetry,
+} from "./retry";
export {
ConflictError,
diff --git a/packages/services/src/page-subscriber/__tests__/page-subscriber.test.ts b/packages/services/src/page-subscriber/__tests__/page-subscriber.test.ts
index 903fb6d42..abbfd66c4 100644
--- a/packages/services/src/page-subscriber/__tests__/page-subscriber.test.ts
+++ b/packages/services/src/page-subscriber/__tests__/page-subscriber.test.ts
@@ -8,9 +8,10 @@ import {
test,
} from "bun:test";
-import { and, db, eq, sql } from "@openstatus/db";
+import { and, db, eq } from "@openstatus/db";
import {
auditLog,
+ page,
pageSubscriber,
pageSubscriberToPageComponent,
} from "@openstatus/db/src/schema";
@@ -24,7 +25,10 @@ import {
upsertSelfSignupSubscriber,
verifySelfSignupSubscriber,
} from "..";
-import { SEEDED_WORKSPACE_TEAM_ID } from "../../../test/fixtures";
+import {
+ SEEDED_WORKSPACE_FREE_ID,
+ SEEDED_WORKSPACE_TEAM_ID,
+} from "../../../test/fixtures";
import {
clearAuditLog,
expectAuditRow,
@@ -286,23 +290,26 @@ describe("upsertSelfSignupSubscriber", () => {
).rejects.toThrow();
});
- // Plan-gate: temporarily flip workspace 1's `status-subscribers` to
- // false, hit the function, then restore. Mirrors the pattern in
- // `apps/server/.../status-page.test.ts`.
test("rejects when the workspace plan disables status-subscribers", async () => {
- await db.run(
- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."status-subscribers"', json('false')) WHERE id = ${SEEDED_WORKSPACE_TEAM_ID}`,
- );
+ const freePage = await db
+ .insert(page)
+ .values({
+ workspaceId: SEEDED_WORKSPACE_FREE_ID,
+ title: "plan-gate",
+ description: "plan-gate",
+ slug: `plan-gate-${Date.now()}`,
+ customDomain: "",
+ })
+ .returning()
+ .get();
try {
await expect(
upsertSelfSignupSubscriber({
- input: { email: EMAILS.planGate, pageId: PAGE_ID },
+ input: { email: EMAILS.planGate, pageId: freePage.id },
}),
).rejects.toThrow("Upgrade to use status subscribers");
} finally {
- await db.run(
- sql`UPDATE workspace SET limits = json_set(COALESCE(limits, '{}'), '$."status-subscribers"', json('true')) WHERE id = ${SEEDED_WORKSPACE_TEAM_ID}`,
- );
+ await db.delete(page).where(eq(page.id, freePage.id));
}
});
});
diff --git a/packages/services/src/retry.ts b/packages/services/src/retry.ts
index 73985a057..98bd7dee8 100644
--- a/packages/services/src/retry.ts
+++ b/packages/services/src/retry.ts
@@ -1,5 +1,6 @@
const RETRYABLE_CODES = new Set(["SQLITE_BUSY", "SQLITE_LOCKED"]);
const RETRYABLE_MESSAGE = /database is (locked|busy)/i;
+const TRANSIENT_SERVER_MESSAGE = /Server returned HTTP status 5\d\d/i;
const MAX_ATTEMPTS = 5;
const BASE_DELAY_MS = 25;
const MAX_DELAY_MS = 400;
@@ -17,6 +18,10 @@ function hasCause(value: object): value is { cause: unknown } {
return "cause" in value;
}
+function hasStatus(value: object): value is { status: unknown } {
+ return "status" in value;
+}
+
export function isRetryableDbError(err: unknown): boolean {
let current: unknown = err;
for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {
@@ -34,6 +39,28 @@ export function isRetryableDbError(err: unknown): boolean {
return false;
}
+// Transient libSQL/Turso 5xx. Safe to retry only for idempotent reads —
+// a 502 may land after a write partially applied. libsql maps every
+// HttpServerError to code "SERVER_ERROR" regardless of status, so gate on the
+// 5xx status (carried on the HttpServerError cause) or the 5xx message — never
+// the bare code, which also covers non-retryable 4xx (bad token, db not found).
+export function isTransientServerError(err: unknown): boolean {
+ let current: unknown = err;
+ for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {
+ if (typeof current !== "object" || current === null) return false;
+
+ if (hasStatus(current) && typeof current.status === "number") {
+ if (current.status >= 500 && current.status <= 599) return true;
+ }
+ if (hasMessage(current) && typeof current.message === "string") {
+ if (TRANSIENT_SERVER_MESSAGE.test(current.message)) return true;
+ }
+ if (!hasCause(current) || current.cause === current) return false;
+ current = current.cause;
+ }
+ return false;
+}
+
function backoffDelayMs(attempt: number): number {
const cap = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** attempt);
return Math.random() * cap;
@@ -41,12 +68,15 @@ function backoffDelayMs(attempt: number): number {
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-export async function withBusyRetry<T>(fn: () => Promise<T>): Promise<T> {
+export async function withBusyRetry<T>(
+ fn: () => Promise<T>,
+ isRetryable: (err: unknown) => boolean = isRetryableDbError,
+): Promise<T> {
for (let attempt = 0; ; attempt++) {
try {
return await fn();
} catch (err) {
- if (attempt >= MAX_ATTEMPTS - 1 || !isRetryableDbError(err)) throw err;
+ if (attempt >= MAX_ATTEMPTS - 1 || !isRetryable(err)) throw err;
await sleep(backoffDelayMs(attempt));
}
}