Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
Add timeouts to checker requests to prevent hanging (#2379)
Maximilian Kaske701e31198eb8
diff --git a/apps/server/src/libs/checker/utils.ts b/apps/server/src/libs/checker/utils.tsindex 66c37f990..01748d183 100644--- a/apps/server/src/libs/checker/utils.ts+++ b/apps/server/src/libs/checker/utils.ts@@ -66,6 +66,14 @@ export function getCheckerPayload(}}+// Bounds outbound checker fetches so a slow check can't pin API requests open+// indefinitely: the monitor's own timeout plus a buffer for checker overhead.+export function getCheckerTimeout(+ monitor: z.infer<typeof selectMonitorSchema>,+): number {+ return (monitor.timeout ?? 45_000) + 15_000;+}+export function getCheckerUrl(monitor: z.infer<typeof selectMonitorSchema>,opts: { trigger?: "api" | "cron"; data?: boolean } = {diff --git a/apps/server/src/routes/rpc/handlers/monitor/index.ts b/apps/server/src/routes/rpc/handlers/monitor/index.tsindex e6a4522b7..2151e55d8 100644--- a/apps/server/src/routes/rpc/handlers/monitor/index.ts+++ b/apps/server/src/routes/rpc/handlers/monitor/index.ts@@ -31,7 +31,11 @@ import {} from "@openstatus/services/monitor";import { env } from "../../../../env";-import { getCheckerPayload, getCheckerUrl } from "../../../../libs/checker";+import {+ getCheckerPayload,+ getCheckerTimeout,+ getCheckerUrl,+} from "../../../../libs/checker";import { toConnectError, toServiceCtx } from "../../adapter";import { getRpcContext } from "../../interceptors";import {@@ -566,6 +570,7 @@ export const monitorServiceImpl: ServiceImpl<typeof MonitorService> = {},method: "POST",body: JSON.stringify(payload),+ signal: AbortSignal.timeout(getCheckerTimeout(row)),});}),);diff --git a/apps/server/src/routes/v1/check/http/post.ts b/apps/server/src/routes/v1/check/http/post.tsindex 1bf0b3716..99f34b433 100644--- a/apps/server/src/routes/v1/check/http/post.ts+++ b/apps/server/src/routes/v1/check/http/post.ts@@ -92,6 +92,8 @@ export function registerHTTPPostCheck(api: typeof checkApi) {}, {}),body: input.body ? input.body : undefined,}),+ // No per-check timeout in the input schema; bound with the checker default.+ signal: AbortSignal.timeout(60_000),});currentFetch.push(r);}diff --git a/apps/server/src/routes/v1/monitors/run/post.ts b/apps/server/src/routes/v1/monitors/run/post.tsindex fe88b8443..c49a97821 100644--- a/apps/server/src/routes/v1/monitors/run/post.ts+++ b/apps/server/src/routes/v1/monitors/run/post.ts@@ -3,7 +3,11 @@ import { getLogger } from "@logtape/logtape";import { and, eq, gte, isNull, sql } from "@openstatus/db";import { env } from "@/env";-import { getCheckerPayload, getCheckerUrl } from "@/libs/checker";+import {+ getCheckerPayload,+ getCheckerTimeout,+ getCheckerUrl,+} from "@/libs/checker";import { openApiErrorResponses } from "@/libs/errors";const logger = getLogger("api-server");@@ -145,11 +149,23 @@ export function registerRunMonitor(api: typeof monitorsApi) {},method: "POST",body: JSON.stringify(payload),+ signal: AbortSignal.timeout(getCheckerTimeout(row)),});allResult.push(result);}if (noWait) {+ // The promises keep running after the response; an uncaught rejection+ // here would crash the Deno process, so swallow and log failures.+ for (const result of allResult) {+ result.catch((error: unknown) => {+ logger.warn("Checker request failed", {+ monitor_id: id,+ workspace_id: workspaceId,+ error: error instanceof Error ? error.message : String(error),+ });+ });+ }return c.json([], 200);}diff --git a/apps/server/src/routes/v1/monitors/trigger/post.ts b/apps/server/src/routes/v1/monitors/trigger/post.tsindex 4bdba9e3f..3e502967c 100644--- a/apps/server/src/routes/v1/monitors/trigger/post.ts+++ b/apps/server/src/routes/v1/monitors/trigger/post.ts@@ -9,7 +9,11 @@ import { selectMonitorSchema } from "@openstatus/db/src/schema/monitors/validatiimport { HTTPException } from "hono/http-exception";import { env } from "@/env";-import { getCheckerPayload, getCheckerUrl } from "@/libs/checker";+import {+ getCheckerPayload,+ getCheckerTimeout,+ getCheckerUrl,+} from "@/libs/checker";import { OpenStatusApiError, openApiErrorResponses } from "@/libs/errors";import type { monitorsApi } from "..";@@ -146,6 +150,7 @@ export function registerTriggerMonitor(api: typeof monitorsApi) {},method: "POST",body: JSON.stringify(payload),+ signal: AbortSignal.timeout(getCheckerTimeout(row)),});allResult.push(result);diff --git a/apps/server/src/serve.ts b/apps/server/src/serve.tsindex b280c1cf5..9080cb04f 100644--- a/apps/server/src/serve.ts+++ b/apps/server/src/serve.ts@@ -5,6 +5,17 @@ import { app } from "./index";const port = 3000;+// Deno terminates the process on unhandled rejections; log instead so one+// stray background promise (e.g. a fire-and-forget fetch) can't kill the instance.+globalThis.addEventListener("unhandledrejection", (event) => {+ const reason = event.reason;+ getLogger("api-server").error("Unhandled promise rejection", {+ reason: reason instanceof Error ? reason.message : String(reason),+ stack: reason instanceof Error ? reason.stack : undefined,+ });+ event.preventDefault();+});+getLogger("api-server").info("Starting server", {port,environment: env.NODE_ENV,diff --git a/apps/server/src/utils/random-promise.ts b/apps/server/src/utils/random-promise.tsdeleted file mode 100644index 7fdfb6933..000000000--- a/apps/server/src/utils/random-promise.ts+++ /dev/null@@ -1,13 +0,0 @@-export function fakePromiseWithRandomResolve() {- return new Promise((resolve, reject) => {- const randomTime = Math.floor(Math.random() * 1000);- setTimeout(() => {- const shouldResolve = Math.random() < 1; // 0.5- if (shouldResolve) {- resolve("Promise resolved successfully.");- } else {- reject(new Error("Promise rejected."));- }- }, randomTime);- });-}diff --git a/packages/db/src/schema/monitors/validation.ts b/packages/db/src/schema/monitors/validation.tsindex 9e2458662..a6774e948 100644--- a/packages/db/src/schema/monitors/validation.ts+++ b/packages/db/src/schema/monitors/validation.ts@@ -43,7 +43,8 @@ export const selectMonitorSchema = createSelectSchema(monitor, {periodicity: monitorPeriodicitySchema.prefault("10m"),status: monitorStatusSchema.prefault("active"),jobType: monitorJobTypesSchema.prefault("http"),- timeout: z.number().prefault(45),+ // milliseconds, matches the column default+ timeout: z.number().prefault(45_000),followRedirects: z.boolean().prefault(true),retry: z.number().prefault(3),regions: regionsToArraySchema.prefault([]),