Skip to content

Open-source status page, incident response and uptime monitoring

Add timeouts to checker requests to prevent hanging (#2379)

Maximilian Kaske701e31198eb8
diff --git a/apps/server/src/libs/checker/utils.ts b/apps/server/src/libs/checker/utils.ts
index 66c37f990..01748d183 100644
--- a/apps/server/src/libs/checker/utils.ts
+++ b/apps/server/src/libs/checker/utils.ts
@@ -66,6 +66,14 @@ export function getCheckerPayload(
}
}
+// Bounds outbound checker fetches so a slow check can't pin API requests open
+// indefinitely: the monitor's own timeout plus a buffer for checker overhead.
+export function getCheckerTimeout(
+ monitor: z.infer<typeof selectMonitorSchema>,
+): number {
+ return (monitor.timeout ?? 45_000) + 15_000;
+}
+
export function getCheckerUrl(
monitor: z.infer<typeof selectMonitorSchema>,
opts: { trigger?: "api" | "cron"; data?: boolean } = {
diff --git a/apps/server/src/routes/rpc/handlers/monitor/index.ts b/apps/server/src/routes/rpc/handlers/monitor/index.ts
index e6a4522b7..2151e55d8 100644
--- a/apps/server/src/routes/rpc/handlers/monitor/index.ts
+++ b/apps/server/src/routes/rpc/handlers/monitor/index.ts
@@ -31,7 +31,11 @@ import {
} from "@openstatus/services/monitor";
import { env } from "../../../../env";
-import { getCheckerPayload, getCheckerUrl } from "../../../../libs/checker";
+import {
+ getCheckerPayload,
+ getCheckerTimeout,
+ getCheckerUrl,
+} from "../../../../libs/checker";
import { toConnectError, toServiceCtx } from "../../adapter";
import { getRpcContext } from "../../interceptors";
import {
@@ -566,6 +570,7 @@ export const monitorServiceImpl: ServiceImpl<typeof MonitorService> = {
},
method: "POST",
body: JSON.stringify(payload),
+ signal: AbortSignal.timeout(getCheckerTimeout(row)),
});
}),
);
diff --git a/apps/server/src/routes/v1/check/http/post.ts b/apps/server/src/routes/v1/check/http/post.ts
index 1bf0b3716..99f34b433 100644
--- a/apps/server/src/routes/v1/check/http/post.ts
+++ b/apps/server/src/routes/v1/check/http/post.ts
@@ -92,6 +92,8 @@ export function registerHTTPPostCheck(api: typeof checkApi) {
}, {}),
body: input.body ? input.body : undefined,
}),
+ // No per-check timeout in the input schema; bound with the checker default.
+ signal: AbortSignal.timeout(60_000),
});
currentFetch.push(r);
}
diff --git a/apps/server/src/routes/v1/monitors/run/post.ts b/apps/server/src/routes/v1/monitors/run/post.ts
index fe88b8443..c49a97821 100644
--- a/apps/server/src/routes/v1/monitors/run/post.ts
+++ b/apps/server/src/routes/v1/monitors/run/post.ts
@@ -3,7 +3,11 @@ import { getLogger } from "@logtape/logtape";
import { and, eq, gte, isNull, sql } from "@openstatus/db";
import { env } from "@/env";
-import { getCheckerPayload, getCheckerUrl } from "@/libs/checker";
+import {
+ getCheckerPayload,
+ getCheckerTimeout,
+ getCheckerUrl,
+} from "@/libs/checker";
import { openApiErrorResponses } from "@/libs/errors";
const logger = getLogger("api-server");
@@ -145,11 +149,23 @@ export function registerRunMonitor(api: typeof monitorsApi) {
},
method: "POST",
body: JSON.stringify(payload),
+ signal: AbortSignal.timeout(getCheckerTimeout(row)),
});
allResult.push(result);
}
if (noWait) {
+ // The promises keep running after the response; an uncaught rejection
+ // here would crash the Deno process, so swallow and log failures.
+ for (const result of allResult) {
+ result.catch((error: unknown) => {
+ logger.warn("Checker request failed", {
+ monitor_id: id,
+ workspace_id: workspaceId,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ });
+ }
return c.json([], 200);
}
diff --git a/apps/server/src/routes/v1/monitors/trigger/post.ts b/apps/server/src/routes/v1/monitors/trigger/post.ts
index 4bdba9e3f..3e502967c 100644
--- a/apps/server/src/routes/v1/monitors/trigger/post.ts
+++ b/apps/server/src/routes/v1/monitors/trigger/post.ts
@@ -9,7 +9,11 @@ import { selectMonitorSchema } from "@openstatus/db/src/schema/monitors/validati
import { HTTPException } from "hono/http-exception";
import { env } from "@/env";
-import { getCheckerPayload, getCheckerUrl } from "@/libs/checker";
+import {
+ getCheckerPayload,
+ getCheckerTimeout,
+ getCheckerUrl,
+} from "@/libs/checker";
import { OpenStatusApiError, openApiErrorResponses } from "@/libs/errors";
import type { monitorsApi } from "..";
@@ -146,6 +150,7 @@ export function registerTriggerMonitor(api: typeof monitorsApi) {
},
method: "POST",
body: JSON.stringify(payload),
+ signal: AbortSignal.timeout(getCheckerTimeout(row)),
});
allResult.push(result);
diff --git a/apps/server/src/serve.ts b/apps/server/src/serve.ts
index b280c1cf5..9080cb04f 100644
--- a/apps/server/src/serve.ts
+++ b/apps/server/src/serve.ts
@@ -5,6 +5,17 @@ import { app } from "./index";
const port = 3000;
+// Deno terminates the process on unhandled rejections; log instead so one
+// stray background promise (e.g. a fire-and-forget fetch) can't kill the instance.
+globalThis.addEventListener("unhandledrejection", (event) => {
+ const reason = event.reason;
+ getLogger("api-server").error("Unhandled promise rejection", {
+ reason: reason instanceof Error ? reason.message : String(reason),
+ stack: reason instanceof Error ? reason.stack : undefined,
+ });
+ event.preventDefault();
+});
+
getLogger("api-server").info("Starting server", {
port,
environment: env.NODE_ENV,
diff --git a/apps/server/src/utils/random-promise.ts b/apps/server/src/utils/random-promise.ts
deleted file mode 100644
index 7fdfb6933..000000000
--- a/apps/server/src/utils/random-promise.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-export function fakePromiseWithRandomResolve() {
- return new Promise((resolve, reject) => {
- const randomTime = Math.floor(Math.random() * 1000);
- setTimeout(() => {
- const shouldResolve = Math.random() < 1; // 0.5
- if (shouldResolve) {
- resolve("Promise resolved successfully.");
- } else {
- reject(new Error("Promise rejected."));
- }
- }, randomTime);
- });
-}
diff --git a/packages/db/src/schema/monitors/validation.ts b/packages/db/src/schema/monitors/validation.ts
index 9e2458662..a6774e948 100644
--- a/packages/db/src/schema/monitors/validation.ts
+++ b/packages/db/src/schema/monitors/validation.ts
@@ -43,7 +43,8 @@ export const selectMonitorSchema = createSelectSchema(monitor, {
periodicity: monitorPeriodicitySchema.prefault("10m"),
status: monitorStatusSchema.prefault("active"),
jobType: monitorJobTypesSchema.prefault("http"),
- timeout: z.number().prefault(45),
+ // milliseconds, matches the column default
+ timeout: z.number().prefault(45_000),
followRedirects: z.boolean().prefault(true),
retry: z.number().prefault(3),
regions: regionsToArraySchema.prefault([]),