Skip to content

Open-source status page, incident response and uptime monitoring

fix: ci tests (#2181)

Colin Ozanne3c1e2c928f97
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 77888252b..e0da58cee 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,7 +14,7 @@ jobs:
timeout-minutes: 15
services:
sqld:
- image: ghcr.io/tursodatabase/libsql-server:latest
+ image: ghcr.io/tursodatabase/libsql-server:e4beaca
ports:
- 8080:8080
# env:
diff --git a/packages/api/src/router/import.test.ts b/packages/api/src/router/import.test.ts
index de435ba0d..46eb31370 100644
--- a/packages/api/src/router/import.test.ts
+++ b/packages/api/src/router/import.test.ts
@@ -19,6 +19,7 @@ import {
MOCK_PAGES,
MOCK_SUBSCRIBERS,
} from "@openstatus/importers/statuspage/fixtures";
+import { clearAuditLogFor } from "@openstatus/services/test/helpers";
import { edgeRouter } from "../edge";
import { createInnerTRPCContext } from "../trpc";
@@ -121,8 +122,21 @@ async function cleanup() {
);
}
- // 3. statusReportUpdate (via statusReportId)
+ // 3. statusReportUpdate (via statusReportId). Capture ids first so we
+ // can wipe their audit rows — `statusReportUpdate.id` recycles after
+ // delete, and an orphan audit row would re-attribute a later
+ // unrelated update.
+ let statusReportUpdateIds: number[] = [];
if (createdIds.statusReports.length > 0) {
+ statusReportUpdateIds = (
+ await db
+ .select({ id: statusReportUpdate.id })
+ .from(statusReportUpdate)
+ .where(
+ inArray(statusReportUpdate.statusReportId, createdIds.statusReports),
+ )
+ .all()
+ ).map((r) => r.id);
await db
.delete(statusReportUpdate)
.where(
@@ -163,6 +177,35 @@ async function cleanup() {
await db.delete(page).where(inArray(page.id, createdIds.pages));
}
+ // 9. audit_log: the importer service emits one audit row per created
+ // entity. The entities are gone but the rows are not — and
+ // INTEGER PRIMARY KEY ids recycle, so a later test inserting into the
+ // same table can land on an id that already has an audit row,
+ // inheriting its actor attribution. See docs/adr/test-audit-cleanup.md.
+ await Promise.all([
+ clearAuditLogFor({ entityType: "page", entityIds: createdIds.pages }),
+ clearAuditLogFor({
+ entityType: "page_component_group",
+ entityIds: createdIds.componentGroups,
+ }),
+ clearAuditLogFor({
+ entityType: "page_component",
+ entityIds: createdIds.components,
+ }),
+ clearAuditLogFor({
+ entityType: "status_report",
+ entityIds: createdIds.statusReports,
+ }),
+ clearAuditLogFor({
+ entityType: "status_report_update",
+ entityIds: statusReportUpdateIds,
+ }),
+ clearAuditLogFor({
+ entityType: "maintenance",
+ entityIds: createdIds.maintenances,
+ }),
+ ]);
+
// Reset trackers
createdIds.pages = [];
createdIds.componentGroups = [];
diff --git a/packages/api/src/router/maintenance.test.ts b/packages/api/src/router/maintenance.test.ts
index 273f1cc38..fccbd9d43 100644
--- a/packages/api/src/router/maintenance.test.ts
+++ b/packages/api/src/router/maintenance.test.ts
@@ -6,6 +6,7 @@ import {
page,
pageComponent,
} from "@openstatus/db/src/schema";
+import { clearAuditLogFor } from "@openstatus/services/test/helpers";
import { TRPCError } from "@trpc/server";
import { edgeRouter } from "../edge";
@@ -15,6 +16,14 @@ let otherWorkspaceMaintenanceId: number;
let otherWorkspacePageId: number;
let otherWorkspaceComponentId: number;
+// Track maintenance ids the tRPC service writes / mutates. Without this
+// the `maintenance.create` / `maintenance.update` audit rows outlive
+// the row itself and — because INTEGER PRIMARY KEY recycles — a later
+// test inserting into `maintenance` can land on the orphan's id and
+// inherit its actor attribution. See docs/adr/test-audit-cleanup.md.
+const createdMaintenanceIds: number[] = [];
+const updatedMaintenanceIds: number[] = [];
+
beforeAll(async () => {
const p = await db
.insert(page)
@@ -82,6 +91,19 @@ afterAll(async () => {
.delete(pageComponent)
.where(eq(pageComponent.id, otherWorkspaceComponentId));
await db.delete(page).where(eq(page.id, otherWorkspacePageId));
+
+ if (createdMaintenanceIds.length > 0) {
+ await clearAuditLogFor({
+ entityType: "maintenance",
+ entityIds: createdMaintenanceIds,
+ });
+ }
+ if (updatedMaintenanceIds.length > 0) {
+ await clearAuditLogFor({
+ entityType: "maintenance",
+ entityIds: updatedMaintenanceIds,
+ });
+ }
});
test("maintenance.update rejects maintenance from another workspace", async () => {
@@ -136,6 +158,7 @@ test("maintenance.update succeeds for own workspace maintenance", async () => {
endDate: new Date(Date.now() + 7_200_000),
pageComponents: [1],
});
+ updatedMaintenanceIds.push(1);
const updated = await db.query.maintenance.findFirst({
where: eq(maintenance.id, 1),
@@ -187,6 +210,7 @@ test("maintenance.new succeeds for own workspace page", async () => {
});
expect(result).toBeDefined();
+ createdMaintenanceIds.push(result.id);
// Cleanup
await db
diff --git a/packages/api/src/router/statusReport.test.ts b/packages/api/src/router/statusReport.test.ts
index e4aa12f12..9786c3d45 100644
--- a/packages/api/src/router/statusReport.test.ts
+++ b/packages/api/src/router/statusReport.test.ts
@@ -1,11 +1,13 @@
import { afterAll, beforeAll, expect, test } from "bun:test";
-import { db, eq } from "@openstatus/db";
+import { db, eq, inArray } from "@openstatus/db";
import {
page,
pageComponent,
statusReport,
+ statusReportUpdate,
statusReportsToPageComponents,
} from "@openstatus/db/src/schema";
+import { clearAuditLogFor } from "@openstatus/services/test/helpers";
import { TRPCError } from "@trpc/server";
import { edgeRouter } from "../edge";
@@ -15,6 +17,16 @@ let otherWorkspacePageId: number;
let otherWorkspaceReportId: number;
let otherWorkspaceComponentId: number;
+// Track ids the service writes so afterAll can scrub the resulting
+// audit rows. `caller.statusReport.create` returns the initial-update
+// row (not the report), so the original cleanup-via-`result.id` never
+// reached the parent — leaving an orphan report and an orphan audit row.
+// SQLite recycles INTEGER PRIMARY KEY ids on delete, so a later test
+// inserting a status_report could land on the orphan's id and inherit
+// its `actor_type=user` attribution. See docs/adr/test-audit-cleanup.md.
+const createdReportIds: number[] = [];
+const updatedReportIds: number[] = [];
+
beforeAll(async () => {
const p = await db
.insert(page)
@@ -75,6 +87,46 @@ afterAll(async () => {
.delete(pageComponent)
.where(eq(pageComponent.id, otherWorkspaceComponentId));
await db.delete(page).where(eq(page.id, otherWorkspacePageId));
+
+ // Drop entities + audit rows the tRPC service emitted during the
+ // tests. Children (status_report_update + join rows) first; then the
+ // parent; then `audit_log` so no orphan rows survive the suite.
+ if (createdReportIds.length > 0) {
+ const updateRows = await db
+ .select({ id: statusReportUpdate.id })
+ .from(statusReportUpdate)
+ .where(inArray(statusReportUpdate.statusReportId, createdReportIds))
+ .all();
+ const updateIds = updateRows.map((r) => r.id);
+ await db
+ .delete(statusReportsToPageComponents)
+ .where(
+ inArray(statusReportsToPageComponents.statusReportId, createdReportIds),
+ );
+ await db
+ .delete(statusReportUpdate)
+ .where(inArray(statusReportUpdate.statusReportId, createdReportIds));
+ await db
+ .delete(statusReport)
+ .where(inArray(statusReport.id, createdReportIds));
+ await clearAuditLogFor({
+ entityType: "status_report",
+ entityIds: createdReportIds,
+ });
+ await clearAuditLogFor({
+ entityType: "status_report_update",
+ entityIds: updateIds,
+ });
+ }
+ // `updateStatus` only mutates an existing report; the row stays, but
+ // its `status_report.update` audit entry is the test's only side
+ // effect and must not outlive the suite.
+ if (updatedReportIds.length > 0) {
+ await clearAuditLogFor({
+ entityType: "status_report",
+ entityIds: updatedReportIds,
+ });
+ }
});
test("statusReport.create rejects pageId from another workspace", async () => {
@@ -122,8 +174,19 @@ test("statusReport.create succeeds for own workspace page", async () => {
expect(result).toBeDefined();
- // Clean up
- await db.delete(statusReport).where(eq(statusReport.id, result.id));
+ // `result.id` is the initial status_report_update id, not the parent
+ // report — recover the report id via the FK so afterAll can scrub both
+ // it and its audit row.
+ if (result) {
+ const updateRow = await db
+ .select({ statusReportId: statusReportUpdate.statusReportId })
+ .from(statusReportUpdate)
+ .where(eq(statusReportUpdate.id, result.id))
+ .get();
+ if (updateRow?.statusReportId != null) {
+ createdReportIds.push(updateRow.statusReportId);
+ }
+ }
});
test("statusReport.updateStatus rejects report from another workspace", async () => {
@@ -174,6 +237,7 @@ test("statusReport.updateStatus succeeds for own workspace report", async () =>
title: "Updated Title",
status: "resolved",
});
+ updatedReportIds.push(1);
const updated = await db.query.statusReport.findFirst({
where: eq(statusReport.id, 1),
diff --git a/packages/services/src/__tests__/retry.test.ts b/packages/services/src/__tests__/retry.test.ts
new file mode 100644
index 000000000..e524956b3
--- /dev/null
+++ b/packages/services/src/__tests__/retry.test.ts
@@ -0,0 +1,74 @@
+import { describe, expect, test } from "bun:test";
+
+import { NotFoundError } from "../errors";
+import { isRetryableDbError, withBusyRetry } from "../retry";
+
+describe("isRetryableDbError", () => {
+ test("true for SQLITE_BUSY code", () => {
+ expect(isRetryableDbError({ code: "SQLITE_BUSY" })).toBe(true);
+ });
+
+ test("true for SQLITE_LOCKED code", () => {
+ expect(isRetryableDbError({ code: "SQLITE_LOCKED" })).toBe(true);
+ });
+
+ test("true for nested cause carrying the code", () => {
+ const err = new Error("Failed query");
+ (err as Error & { cause: unknown }).cause = { code: "SQLITE_BUSY" };
+ expect(isRetryableDbError(err)).toBe(true);
+ });
+
+ test("true for message-only match without a code", () => {
+ expect(
+ isRetryableDbError(new Error("SQLite error: database is locked")),
+ ).toBe(true);
+ });
+
+ test("false for an unrelated error", () => {
+ expect(isRetryableDbError(new Error("nope"))).toBe(false);
+ });
+
+ test("false for a NotFoundError", () => {
+ expect(isRetryableDbError(new NotFoundError("page", 1))).toBe(false);
+ });
+
+ test("false for non-object values", () => {
+ expect(isRetryableDbError(null)).toBe(false);
+ expect(isRetryableDbError("SQLITE_BUSY")).toBe(false);
+ });
+});
+
+describe("withBusyRetry", () => {
+ test("resolves once a transient busy error clears", async () => {
+ let attempts = 0;
+ const result = await withBusyRetry(async () => {
+ attempts++;
+ if (attempts < 3) throw { code: "SQLITE_BUSY" };
+ return "ok";
+ });
+ expect(result).toBe("ok");
+ expect(attempts).toBe(3);
+ });
+
+ test("gives up after 5 attempts and rethrows the original error", async () => {
+ let attempts = 0;
+ const original = { code: "SQLITE_BUSY", marker: Symbol("x") };
+ const promise = withBusyRetry(async () => {
+ attempts++;
+ throw original;
+ });
+ await expect(promise).rejects.toBe(original);
+ expect(attempts).toBe(5);
+ });
+
+ test("does not retry a non-retryable error", async () => {
+ let attempts = 0;
+ const original = new NotFoundError("page", 1);
+ const promise = withBusyRetry(async () => {
+ attempts++;
+ throw original;
+ });
+ await expect(promise).rejects.toBe(original);
+ expect(attempts).toBe(1);
+ });
+});
diff --git a/packages/services/src/context.ts b/packages/services/src/context.ts
index 1f77bfb11..73dc7375e 100644
--- a/packages/services/src/context.ts
+++ b/packages/services/src/context.ts
@@ -1,6 +1,7 @@
import { SQLiteTransaction, db as defaultDb, is } from "@openstatus/db";
import type { Scope, Workspace } from "@openstatus/db/src/schema";
import { OSTinybird } from "@openstatus/tinybird";
+import { withBusyRetry } from "./retry";
// `@openstatus/db` does not export named DrizzleClient / DrizzleTx types today,
// so we derive them from the db export and re-export from here.
@@ -43,7 +44,7 @@ export async function withTransaction<T>(
): Promise<T> {
const db = ctx.db ?? defaultDb;
if (isTx(db)) return fn(db);
- return (db as DrizzleClient).transaction(fn);
+ return withBusyRetry(() => (db as DrizzleClient).transaction(fn));
}
/**
diff --git a/packages/services/src/index.ts b/packages/services/src/index.ts
index df9582d4e..a07c511a6 100644
--- a/packages/services/src/index.ts
+++ b/packages/services/src/index.ts
@@ -11,6 +11,8 @@ export {
withTransaction,
} from "./context";
+export { isRetryableDbError, withBusyRetry } from "./retry";
+
export {
ConflictError,
ForbiddenError,
diff --git a/packages/services/src/retry.ts b/packages/services/src/retry.ts
new file mode 100644
index 000000000..73985a057
--- /dev/null
+++ b/packages/services/src/retry.ts
@@ -0,0 +1,53 @@
+const RETRYABLE_CODES = new Set(["SQLITE_BUSY", "SQLITE_LOCKED"]);
+const RETRYABLE_MESSAGE = /database is (locked|busy)/i;
+const MAX_ATTEMPTS = 5;
+const BASE_DELAY_MS = 25;
+const MAX_DELAY_MS = 400;
+const MAX_CAUSE_DEPTH = 10;
+
+function hasCode(value: object): value is { code: unknown } {
+ return "code" in value;
+}
+
+function hasMessage(value: object): value is { message: unknown } {
+ return "message" in value;
+}
+
+function hasCause(value: object): value is { cause: unknown } {
+ return "cause" in value;
+}
+
+export function isRetryableDbError(err: unknown): boolean {
+ let current: unknown = err;
+ for (let depth = 0; depth < MAX_CAUSE_DEPTH; depth++) {
+ if (typeof current !== "object" || current === null) return false;
+
+ if (hasCode(current) && typeof current.code === "string") {
+ if (RETRYABLE_CODES.has(current.code)) return true;
+ }
+ if (hasMessage(current) && typeof current.message === "string") {
+ if (RETRYABLE_MESSAGE.test(current.message)) return true;
+ }
+ if (!hasCause(current) || current.cause === current) return false;
+ current = current.cause;
+ }
+ return false;
+}
+
+function backoffDelayMs(attempt: number): number {
+ const cap = Math.min(MAX_DELAY_MS, BASE_DELAY_MS * 2 ** attempt);
+ return Math.random() * cap;
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+export async function withBusyRetry<T>(fn: () => Promise<T>): Promise<T> {
+ for (let attempt = 0; ; attempt++) {
+ try {
+ return await fn();
+ } catch (err) {
+ if (attempt >= MAX_ATTEMPTS - 1 || !isRetryableDbError(err)) throw err;
+ await sleep(backoffDelayMs(attempt));
+ }
+ }
+}
diff --git a/packages/services/test/helpers.ts b/packages/services/test/helpers.ts
index eaf7cf1d4..dbdaea08c 100644
--- a/packages/services/test/helpers.ts
+++ b/packages/services/test/helpers.ts
@@ -99,6 +99,37 @@ export async function clearAuditLog(
.catch(() => undefined);
}
+/**
+ * Delete audit_log rows for specific (entityType, entityId) pairs.
+ *
+ * Tests that create entities through services on the committed db and
+ * then delete those entities in cleanup must also call this — otherwise
+ * the audit row outlives the entity, and because SQLite recycles
+ * INTEGER PRIMARY KEY ids after deletes, a later test's freshly-inserted
+ * entity can land on the orphan's id and inherit its actor attribution.
+ * See docs/adr/test-audit-cleanup.md.
+ */
+export async function clearAuditLogFor(args: {
+ entityType: string;
+ entityIds: ReadonlyArray<string | number>;
+ db?: DB;
+}): Promise<void> {
+ if (args.entityIds.length === 0) return;
+ const conn = args.db ?? db;
+ await conn
+ .delete(auditLog)
+ .where(
+ and(
+ eq(auditLog.entityType, args.entityType),
+ inArray(
+ auditLog.entityId,
+ args.entityIds.map((id) => String(id)),
+ ),
+ ),
+ )
+ .catch(() => undefined);
+}
+
/**
* Load a seeded workspace by id (defaults to id=1, the `team` plan fixture).
* Tests that need a fresh workspace should insert one explicitly and clean up;
@@ -257,8 +288,10 @@ export async function expectAuditRow(match: {
row.action === match.action &&
(match.actorType === undefined || row.actorType === match.actorType),
);
- expect(
- hit,
- `expected audit row for ${match.action} on ${match.entityType}#${expectedEntityId}`,
- ).toBeDefined();
+ if (hit === undefined) {
+ throw new Error(
+ `expected audit row for ${match.action} on ${match.entityType}#${expectedEntityId}`,
+ );
+ }
+ expect(hit).toBeDefined();
}