Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
fix: ci tests (#2181)
Colin Ozanne3c1e2c928f97
diff --git a/.github/workflows/test.yml b/.github/workflows/test.ymlindex 77888252b..e0da58cee 100644--- a/.github/workflows/test.yml+++ b/.github/workflows/test.yml@@ -14,7 +14,7 @@ jobs:timeout-minutes: 15services:sqld:- image: ghcr.io/tursodatabase/libsql-server:latest+ image: ghcr.io/tursodatabase/libsql-server:e4beacaports:- 8080:8080# env:diff --git a/packages/api/src/router/import.test.ts b/packages/api/src/router/import.test.tsindex de435ba0d..46eb31370 100644--- a/packages/api/src/router/import.test.ts+++ b/packages/api/src/router/import.test.ts@@ -19,6 +19,7 @@ import {MOCK_PAGES,MOCK_SUBSCRIBERS,} from "@openstatus/importers/statuspage/fixtures";+import { clearAuditLogFor } from "@openstatus/services/test/helpers";import { edgeRouter } from "../edge";import { createInnerTRPCContext } from "../trpc";@@ -121,8 +122,21 @@ async function cleanup() {);}- // 3. statusReportUpdate (via statusReportId)+ // 3. statusReportUpdate (via statusReportId). Capture ids first so we+ // can wipe their audit rows — `statusReportUpdate.id` recycles after+ // delete, and an orphan audit row would re-attribute a later+ // unrelated update.+ let statusReportUpdateIds: number[] = [];if (createdIds.statusReports.length > 0) {+ statusReportUpdateIds = (+ await db+ .select({ id: statusReportUpdate.id })+ .from(statusReportUpdate)+ .where(+ inArray(statusReportUpdate.statusReportId, createdIds.statusReports),+ )+ .all()+ ).map((r) => r.id);await db.delete(statusReportUpdate).where(@@ -163,6 +177,35 @@ async function cleanup() {await db.delete(page).where(inArray(page.id, createdIds.pages));}+ // 9. audit_log: the importer service emits one audit row per created+ // entity. The entities are gone but the rows are not — and+ // INTEGER PRIMARY KEY ids recycle, so a later test inserting into the+ // same table can land on an id that already has an audit row,+ // inheriting its actor attribution. See docs/adr/test-audit-cleanup.md.+ await Promise.all([+ clearAuditLogFor({ entityType: "page", entityIds: createdIds.pages }),+ clearAuditLogFor({+ entityType: "page_component_group",+ entityIds: createdIds.componentGroups,+ }),+ clearAuditLogFor({+ entityType: "page_component",+ entityIds: createdIds.components,+ }),+ clearAuditLogFor({+ entityType: "status_report",+ entityIds: createdIds.statusReports,+ }),+ clearAuditLogFor({+ entityType: "status_report_update",+ entityIds: statusReportUpdateIds,+ }),+ clearAuditLogFor({+ entityType: "maintenance",+ entityIds: createdIds.maintenances,+ }),+ ]);+// Reset trackerscreatedIds.pages = [];createdIds.componentGroups = [];diff --git a/packages/api/src/router/maintenance.test.ts b/packages/api/src/router/maintenance.test.tsindex 273f1cc38..fccbd9d43 100644--- a/packages/api/src/router/maintenance.test.ts+++ b/packages/api/src/router/maintenance.test.ts@@ -6,6 +6,7 @@ import {page,pageComponent,} from "@openstatus/db/src/schema";+import { clearAuditLogFor } from "@openstatus/services/test/helpers";import { TRPCError } from "@trpc/server";import { edgeRouter } from "../edge";@@ -15,6 +16,14 @@ let otherWorkspaceMaintenanceId: number;let otherWorkspacePageId: number;let otherWorkspaceComponentId: number;+// Track maintenance ids the tRPC service writes / mutates. Without this+// the `maintenance.create` / `maintenance.update` audit rows outlive+// the row itself and — because INTEGER PRIMARY KEY recycles — a later+// test inserting into `maintenance` can land on the orphan's id and+// inherit its actor attribution. See docs/adr/test-audit-cleanup.md.+const createdMaintenanceIds: number[] = [];+const updatedMaintenanceIds: number[] = [];+beforeAll(async () => {const p = await db.insert(page)@@ -82,6 +91,19 @@ afterAll(async () => {.delete(pageComponent).where(eq(pageComponent.id, otherWorkspaceComponentId));await db.delete(page).where(eq(page.id, otherWorkspacePageId));++ if (createdMaintenanceIds.length > 0) {+ await clearAuditLogFor({+ entityType: "maintenance",+ entityIds: createdMaintenanceIds,+ });+ }+ if (updatedMaintenanceIds.length > 0) {+ await clearAuditLogFor({+ entityType: "maintenance",+ entityIds: updatedMaintenanceIds,+ });+ }});test("maintenance.update rejects maintenance from another workspace", async () => {@@ -136,6 +158,7 @@ test("maintenance.update succeeds for own workspace maintenance", async () => {endDate: new Date(Date.now() + 7_200_000),pageComponents: [1],});+ updatedMaintenanceIds.push(1);const updated = await db.query.maintenance.findFirst({where: eq(maintenance.id, 1),@@ -187,6 +210,7 @@ test("maintenance.new succeeds for own workspace page", async () => {});expect(result).toBeDefined();+ createdMaintenanceIds.push(result.id);// Cleanupawait dbdiff --git a/packages/api/src/router/statusReport.test.ts b/packages/api/src/router/statusReport.test.tsindex e4aa12f12..9786c3d45 100644--- a/packages/api/src/router/statusReport.test.ts+++ b/packages/api/src/router/statusReport.test.ts@@ -1,11 +1,13 @@import { afterAll, beforeAll, expect, test } from "bun:test";-import { db, eq } from "@openstatus/db";+import { db, eq, inArray } from "@openstatus/db";import {page,pageComponent,statusReport,+ statusReportUpdate,statusReportsToPageComponents,} from "@openstatus/db/src/schema";+import { clearAuditLogFor } from "@openstatus/services/test/helpers";import { TRPCError } from "@trpc/server";import { edgeRouter } from "../edge";@@ -15,6 +17,16 @@ let otherWorkspacePageId: number;let otherWorkspaceReportId: number;let otherWorkspaceComponentId: number;+// Track ids the service writes so afterAll can scrub the resulting+// audit rows. `caller.statusReport.create` returns the initial-update+// row (not the report), so the original cleanup-via-`result.id` never+// reached the parent — leaving an orphan report and an orphan audit row.+// SQLite recycles INTEGER PRIMARY KEY ids on delete, so a later test+// inserting a status_report could land on the orphan's id and inherit+// its `actor_type=user` attribution. See docs/adr/test-audit-cleanup.md.+const createdReportIds: number[] = [];+const updatedReportIds: number[] = [];+beforeAll(async () => {const p = await db.insert(page)@@ -75,6 +87,46 @@ afterAll(async () => {.delete(pageComponent).where(eq(pageComponent.id, otherWorkspaceComponentId));await db.delete(page).where(eq(page.id, otherWorkspacePageId));++ // Drop entities + audit rows the tRPC service emitted during the+ // tests. Children (status_report_update + join rows) first; then the+ // parent; then `audit_log` so no orphan rows survive the suite.+ if (createdReportIds.length > 0) {+ const updateRows = await db+ .select({ id: statusReportUpdate.id })+ .from(statusReportUpdate)+ .where(inArray(statusReportUpdate.statusReportId, createdReportIds))+ .all();+ const updateIds = updateRows.map((r) => r.id);+ await db+ .delete(statusReportsToPageComponents)+ .where(+ inArray(statusReportsToPageComponents.statusReportId, createdReportIds),+ );+ await db+ .delete(statusReportUpdate)+ .where(inArray(statusReportUpdate.statusReportId, createdReportIds));+ await db+ .delete(statusReport)+ .where(inArray(statusReport.id, createdReportIds));+ await clearAuditLogFor({+ entityType: "status_report",+ entityIds: createdReportIds,+ });+ await clearAuditLogFor({+ entityType: "status_report_update",+ entityIds: updateIds,+ });+ }+ // `updateStatus` only mutates an existing report; the row stays, but+ // its `status_report.update` audit entry is the test's only side+ // effect and must not outlive the suite.+ if (updatedReportIds.length > 0) {+ await clearAuditLogFor({+ entityType: "status_report",+ entityIds: updatedReportIds,+ });+ }});test("statusReport.create rejects pageId from another workspace", async () => {@@ -122,8 +174,19 @@ test("statusReport.create succeeds for own workspace page", async () => {expect(result).toBeDefined();- // Clean up- await db.delete(statusReport).where(eq(statusReport.id, result.id));+ // `result.id` is the initial status_report_update id, not the parent+ // report — recover the report id via the FK so afterAll can scrub both+ // it and its audit row.+ if (result) {+ const updateRow = await db+ .select({ statusReportId: statusReportUpdate.statusReportId })+ .from(statusReportUpdate)+ .where(eq(statusReportUpdate.id, result.id))+ .get();+ if (updateRow?.statusReportId != null) {+ createdReportIds.push(updateRow.statusReportId);+ }+ }});test("statusReport.updateStatus rejects report from another workspace", async () => {@@ -174,6 +237,7 @@ test("statusReport.updateStatus succeeds for own workspace report", async () =>title: "Updated Title",status: "resolved",});+ updatedReportIds.push(1);const updated = await db.query.statusReport.findFirst({where: eq(statusReport.id, 1),diff --git a/packages/services/src/__tests__/retry.test.ts b/packages/services/src/__tests__/retry.test.tsnew file mode 100644index 000000000..e524956b3--- /dev/null+++ b/packages/services/src/__tests__/retry.test.ts@@ -0,0 +1,74 @@+import { describe, expect, test } from "bun:test";++import { NotFoundError } from "../errors";+import { isRetryableDbError, withBusyRetry } from "../retry";++describe("isRetryableDbError", () => {+ test("true for SQLITE_BUSY code", () => {+ expect(isRetryableDbError({ code: "SQLITE_BUSY" })).toBe(true);+ });++ test("true for SQLITE_LOCKED code", () => {+ expect(isRetryableDbError({ code: "SQLITE_LOCKED" })).toBe(true);+ });++ test("true for nested cause carrying the code", () => {+ const err = new Error("Failed query");+ (err as Error & { cause: unknown }).cause = { code: "SQLITE_BUSY" };+ expect(isRetryableDbError(err)).toBe(true);+ });++ test("true for message-only match without a code", () => {+ expect(+ isRetryableDbError(new Error("SQLite error: database is locked")),+ ).toBe(true);+ });++ test("false for an unrelated error", () => {+ expect(isRetryableDbError(new Error("nope"))).toBe(false);+ });++ test("false for a NotFoundError", () => {+ expect(isRetryableDbError(new NotFoundError("page", 1))).toBe(false);+ });++ test("false for non-object values", () => {+ expect(isRetryableDbError(null)).toBe(false);+ expect(isRetryableDbError("SQLITE_BUSY")).toBe(false);+ });+});++describe("withBusyRetry", () => {+ test("resolves once a transient busy error clears", async () => {+ let attempts = 0;+ const result = await withBusyRetry(async () => {+ attempts++;+ if (attempts < 3) throw { code: "SQLITE_BUSY" };+ return "ok";+ });+ expect(result).toBe("ok");+ expect(attempts).toBe(3);+ });++ test("gives up after 5 attempts and rethrows the original error", async () => {+ let attempts = 0;+ const original = { code: "SQLITE_BUSY", marker: Symbol("x") };+ const promise = withBusyRetry(async () => {+ attempts++;+ throw original;+ });+ await expect(promise).rejects.toBe(original);+ expect(attempts).toBe(5);+ });++ test("does not retry a non-retryable error", async () => {+ let attempts = 0;+ const original = new NotFoundError("page", 1);+ const promise = withBusyRetry(async () => {+ attempts++;+ throw original;+ });+ await expect(promise).rejects.toBe(original);+ expect(attempts).toBe(1);+ });+});diff --git a/packages/services/src/context.ts b/packages/services/src/context.tsindex 1f77bfb11..73dc7375e 100644--- a/packages/services/src/context.ts+++ b/packages/services/src/context.ts@@ -1,6 +1,7 @@import { SQLiteTransaction, db as defaultDb, is } from "@openstatus/db";import type { Scope, Workspace } from "@openstatus/db/src/schema";import { OSTinybird } from "@openstatus/tinybird";+import { withBusyRetry } from "./retry";// `@openstatus/db` does not export named DrizzleClient / DrizzleTx types today,// so we derive them from the db export and re-export from here.@@ -43,7 +44,7 @@ export async function withTransaction<T>(): Promise<T> {const db = ctx.db ?? defaultDb;if (isTx(db)) return fn(db);- return (db as DrizzleClient).transaction(fn);+ return withBusyRetry(() => (db as DrizzleClient).transaction(fn));}/**diff --git a/packages/services/src/index.ts b/packages/services/src/index.tsindex df9582d4e..a07c511a6 100644--- a/packages/services/src/index.ts+++ b/packages/services/src/index.ts@@ -11,6 +11,8 @@ export {withTransaction,} from "./context";+export { isRetryableDbError, withBusyRetry } from "./retry";+export {ConflictError,ForbiddenError,diff --git a/packages/services/src/retry.ts b/packages/services/src/retry.tsnew file mode 100644index 000000000..73985a057--- /dev/null+++ b/packages/services/src/retry.ts@@ -0,0 +1,53 @@+const RETRYABLE_CODES = new Set(["SQLITE_BUSY", "SQLITE_LOCKED"]);+const RETRYABLE_MESSAGE = /database is (locked|busy)/i;+const MAX_ATTEMPTS = 5;+const BASE_DELAY_MS = 25;+const MAX_DELAY_MS = 400;+const MAX_CAUSE_DEPTH = 10;++function hasCode(value: object): value is { code: unknown } {+ return "code" in value;+}++function hasMessage(value: object): value is { message: unknown } {+ return "message" in value;+}++function hasCause(value: object): value is { cause: unknown } {+ return "cause" in value;+}++export function isRetryableDbError(err: unknown): boolean {+ let current: unknown = err;+ for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {+ if (typeof current !== "object" || current === null) return false;++ if (hasCode(current) && typeof current.code === "string") {+ if (RETRYABLE_CODES.has(current.code)) return true;+ }+ if (hasMessage(current) && typeof current.message === "string") {+ if (RETRYABLE_MESSAGE.test(current.message)) return true;+ }+ if (!hasCause(current) || current.cause === current) return false;+ current = current.cause;+ }+ return false;+}++function backoffDelayMs(attempt: number): number {+ const cap = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** attempt);+ return Math.random() * cap;+}++const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));++export async function withBusyRetry<T>(fn: () => Promise<T>): Promise<T> {+ for (let attempt = 0; ; attempt++) {+ try {+ return await fn();+ } catch (err) {+ if (attempt >= MAX_ATTEMPTS - 1 || !isRetryableDbError(err)) throw err;+ await sleep(backoffDelayMs(attempt));+ }+ }+}diff --git a/packages/services/test/helpers.ts b/packages/services/test/helpers.tsindex eaf7cf1d4..dbdaea08c 100644--- a/packages/services/test/helpers.ts+++ b/packages/services/test/helpers.ts@@ -99,6 +99,37 @@ export async function clearAuditLog(.catch(() => undefined);}+/**+ * Delete audit_log rows for specific (entityType, entityId) pairs.+ *+ * Tests that create entities through services on the committed db and+ * then delete those entities in cleanup must also call this — otherwise+ * the audit row outlives the entity, and because SQLite recycles+ * INTEGER PRIMARY KEY ids after deletes, a later test's freshly-inserted+ * entity can land on the orphan's id and inherit its actor attribution.+ * See docs/adr/test-audit-cleanup.md.+ */+export async function clearAuditLogFor(args: {+ entityType: string;+ entityIds: ReadonlyArray<string | number>;+ db?: DB;+}): Promise<void> {+ if (args.entityIds.length === 0) return;+ const conn = args.db ?? db;+ await conn+ .delete(auditLog)+ .where(+ and(+ eq(auditLog.entityType, args.entityType),+ inArray(+ auditLog.entityId,+ args.entityIds.map((id) => String(id)),+ ),+ ),+ )+ .catch(() => undefined);+}+/*** Load a seeded workspace by id (defaults to id=1, the `team` plan fixture).* Tests that need a fresh workspace should insert one explicitly and clean up;@@ -257,8 +288,10 @@ export async function expectAuditRow(match: {row.action === match.action &&(match.actorType === undefined || row.actorType === match.actorType),);- expect(- hit,- `expected audit row for ${match.action} on ${match.entityType}#${expectedEntityId}`,- ).toBeDefined();+ if (hit === undefined) {+ throw new Error(+ `expected audit row for ${match.action} on ${match.entityType}#${expectedEntityId}`,+ );+ }+ expect(hit).toBeDefined();}