Skip to content

Open-source status page, incident response and uptime monitoring

fix: test checker timeout (#2209)

Maximilian Kaske06bf24d821d1
diff --git a/apps/web/src/app/(landing)/play/checker/[slug]/page.tsx b/apps/web/src/app/(landing)/play/checker/[slug]/page.tsx
index 65c06f078..c54031652 100644
--- a/apps/web/src/app/(landing)/play/checker/[slug]/page.tsx
+++ b/apps/web/src/app/(landing)/play/checker/[slug]/page.tsx
@@ -52,18 +52,22 @@ export async function generateMetadata({
if (!data) return metadata;
- const regions = data.checks.sort((a, b) => a.latency - b.latency);
+ const regions = [...data.checks].sort((a, b) => a.latency - b.latency);
const fastestRegion = regions[0];
const slowestRegion = regions[regions.length - 1];
const TITLE = data.url;
- const DESCRIPTION = `${formatDate(
- new Date(data.timestamp),
- )} | Fastest: ${regionFormatter(fastestRegion.region)} (${latencyFormatter(
- fastestRegion.latency,
- )}) | Slowest: ${regionFormatter(slowestRegion.region)} (${latencyFormatter(
- slowestRegion.latency,
- )})`;
+ // `checks` is empty when every region failed (only the base hash is stored).
+ const DESCRIPTION =
+ fastestRegion && slowestRegion
+ ? `${formatDate(
+ new Date(data.timestamp),
+ )} | Fastest: ${regionFormatter(fastestRegion.region)} (${latencyFormatter(
+ fastestRegion.latency,
+ )}) | Slowest: ${regionFormatter(slowestRegion.region)} (${latencyFormatter(
+ slowestRegion.latency,
+ )})`
+ : `${formatDate(new Date(data.timestamp))} | No successful checks`;
return {
...metadata,
diff --git a/apps/web/src/app/(landing)/play/checker/client.tsx b/apps/web/src/app/(landing)/play/checker/client.tsx
index 1db7ed7cc..5061c6327 100644
--- a/apps/web/src/app/(landing)/play/checker/client.tsx
+++ b/apps/web/src/app/(landing)/play/checker/client.tsx
@@ -132,6 +132,8 @@ export function Form({
startTransition(async () => {
async function fetchAndReadStream() {
let toastId: string | number | undefined;
+ let resultId: string | null = null;
+ let successCount = 0;
try {
toastId = toast.loading("Loading data from regions...", {
duration: Number.POSITIVE_INFINITY,
@@ -171,7 +173,14 @@ export function Form({
clearTimeout(timeoutId);
const reader = response?.body?.getReader();
- if (!reader) return;
+ if (!reader) {
+ toast.error("Failed to read response", {
+ id: toastId,
+ description: "Please try again.",
+ className: "text-destructive!",
+ });
+ return;
+ }
const decoder = new TextDecoder();
let done = false;
@@ -191,15 +200,7 @@ export function Form({
// Store the ID if it's a 32-char hex string
if (is32CharHex(item)) {
setId(item);
- toast.success("Data is available!", {
- id: toastId,
- description: "Learn about the response details.",
- action: {
- label: "Details",
- onClick: () => router.push(`/play/checker/${item}`),
- },
- duration: 4000,
- });
+ resultId = item;
return null;
}
@@ -225,6 +226,7 @@ export function Form({
.filter(notEmpty);
if (results.length > 0) {
+ successCount += results.length;
setValues((prev) => [...prev, ...results]);
toast.loading(
`Checking ${regionFormatter(
@@ -238,13 +240,41 @@ export function Form({
}
}
}
+
+ if (successCount === 0) {
+ toast.error("No region could reach the target", {
+ id: toastId,
+ description: "It may be down or blocking our requests.",
+ className: "text-destructive!",
+ });
+ } else {
+ toast.success("Data is available!", {
+ id: toastId,
+ description: "Learn about the response details.",
+ ...(resultId
+ ? {
+ action: {
+ label: "Details",
+ onClick: () => router.push(`/play/checker/${resultId}`),
+ },
+ }
+ : {}),
+ duration: 4000,
+ });
+ }
} catch (error) {
console.error("Error fetching data:", error);
if (error instanceof Error && error.name === "AbortError") {
toast.error("Request timeout", {
id: toastId,
description:
- "The request took too long and was aborted after 7 seconds.",
+ "The request took too long and was aborted after 10 seconds.",
+ className: "text-destructive!",
+ });
+ } else {
+ toast.error("Something went wrong", {
+ id: toastId,
+ description: "Please try again.",
className: "text-destructive!",
});
}
diff --git a/apps/web/src/app/api/checker/test/http/route.ts b/apps/web/src/app/api/checker/test/http/route.ts
index 653c0c84f..0edf26944 100644
--- a/apps/web/src/app/api/checker/test/http/route.ts
+++ b/apps/web/src/app/api/checker/test/http/route.ts
@@ -3,7 +3,10 @@ import { httpPayloadSchema } from "@openstatus/utils";
import { NextResponse } from "next/server";
import { z } from "zod";
-import { checkRegion } from "../../../../../lib/checker/utils";
+import {
+ TargetUnreachableError,
+ checkRegion,
+} from "../../../../../lib/checker/utils";
import { isAnInvalidTestUrl } from "../../utils";
export const runtime = "edge";
@@ -37,7 +40,10 @@ export async function POST(request: Request) {
return NextResponse.json(res);
} catch (e) {
- console.error(e);
+ // Unreachable/timeout targets are expected; only real bugs should reach Sentry.
+ if (!(e instanceof TargetUnreachableError)) {
+ console.error(e);
+ }
return NextResponse.json({ success: false }, { status: 400 });
}
}
diff --git a/apps/web/src/app/api/checker/test/tcp/route.ts b/apps/web/src/app/api/checker/test/tcp/route.ts
index 11f89c1dd..800cbdcdf 100644
--- a/apps/web/src/app/api/checker/test/tcp/route.ts
+++ b/apps/web/src/app/api/checker/test/tcp/route.ts
@@ -5,6 +5,12 @@ import {
import { NextResponse } from "next/server";
import { z } from "zod";
+import {
+ CHECKER_REQUEST_TIMEOUT_MS,
+ TargetUnreachableError,
+ isTimeoutError,
+} from "@/lib/checker/utils";
+
import { TCPResponse, tcpPayload } from "./schema";
export const runtime = "edge";
@@ -34,36 +40,47 @@ export async function POST(request: Request) {
return NextResponse.json(res);
} catch (e) {
- console.error(e);
+ // Unreachable/timeout targets are expected; only real bugs should reach Sentry.
+ if (!(e instanceof TargetUnreachableError)) {
+ console.error(e);
+ }
return NextResponse.json({ success: false }, { status: 400 });
}
}
async function checkTCP(url: string, region: Region) {
//
- const res = await fetch(`https://checker.openstatus.dev/tcp/${region}`, {
- headers: {
- Authorization: `Basic ${process.env.CRON_SECRET}`,
- "Content-Type": "application/json",
- "fly-prefer-region": region,
- },
- method: "POST",
- body: JSON.stringify({
- uri: url,
- }),
- next: { revalidate: 0 },
- });
+ let res: Response;
+ try {
+ res = await fetch(`https://checker.openstatus.dev/tcp/${region}`, {
+ headers: {
+ Authorization: `Basic ${process.env.CRON_SECRET}`,
+ "Content-Type": "application/json",
+ "fly-prefer-region": region,
+ },
+ method: "POST",
+ body: JSON.stringify({
+ uri: url,
+ }),
+ next: { revalidate: 0 },
+ signal: AbortSignal.timeout(CHECKER_REQUEST_TIMEOUT_MS),
+ });
+ } catch (e) {
+ if (isTimeoutError(e)) {
+ throw new TargetUnreachableError("checker request timed out");
+ }
+ throw e;
+ }
const json = await res.json();
const data = TCPResponse.safeParse(json);
+ // A timeout / unreachable target is an expected outcome, not a bug — throw so
+ // the caller returns 400, but the catch keeps it out of Sentry. A parse miss
+ // here may also be checker schema drift, so warn so it stays observable.
if (!data.success) {
- console.error(res);
- console.error(JSON.stringify(json));
- console.error(
- `something went wrong with request to ${url} error ${data.error.message}`,
- );
- throw new Error(data.error.message);
+ console.warn("Unexpected TCP checker response shape:", json);
+ throw new TargetUnreachableError(data.error.message);
}
return data.data;
diff --git a/apps/web/src/lib/checker/utils.ts b/apps/web/src/lib/checker/utils.ts
index 72ca2ff74..ffe605602 100644
--- a/apps/web/src/lib/checker/utils.ts
+++ b/apps/web/src/lib/checker/utils.ts
@@ -182,6 +182,18 @@ type CheckRegionRequest = {
// API Functions
// ============================================================================
+// A timeout / unreachable target is an expected outcome, not a bug. Callers
+// throw this so a route catch can return 400 while keeping it out of Sentry.
+export class TargetUnreachableError extends Error {}
+
+// Bound the upstream checker request below the client's 10s abort so a stalled
+// checker can't keep the route running to the platform limit.
+export const CHECKER_REQUEST_TIMEOUT_MS = 9_000;
+
+export function isTimeoutError(e: unknown): boolean {
+ return e instanceof Error && e.name === "TimeoutError";
+}
+
export async function checkRegion(
props: CheckRegionRequest,
): Promise<RegionCheckerResponse> {
@@ -209,40 +221,51 @@ export async function checkRegion(
break;
}
- const res = await fetch(endpoint, {
- headers: {
- Authorization: `Basic ${process.env.CRON_SECRET}`,
- "Content-Type": "application/json",
- ...regionHeader,
- },
- method: "POST",
- body: JSON.stringify({
- url,
- method: method || "GET",
- headers: headers?.reduce(
- (acc, { key, value }) => {
- if (!key) return acc; // key === "" is an invalid header
- return { ...acc, [key]: value };
- },
- {} as Record<string, string>,
- ),
- body: body ? body : undefined,
- }),
- signal,
- next: { revalidate: 0 },
- });
+ // A caller-supplied signal is the caller's to interpret (it inspects the raw
+ // TimeoutError); only our own default bound maps to TargetUnreachableError.
+ const usingDefaultTimeout = !signal;
+
+ let res: Response;
+ try {
+ res = await fetch(endpoint, {
+ headers: {
+ Authorization: `Basic ${process.env.CRON_SECRET}`,
+ "Content-Type": "application/json",
+ ...regionHeader,
+ },
+ method: "POST",
+ body: JSON.stringify({
+ url,
+ method: method || "GET",
+ headers: headers?.reduce(
+ (acc, { key, value }) => {
+ if (!key) return acc; // key === "" is an invalid header
+ return { ...acc, [key]: value };
+ },
+ {} as Record<string, string>,
+ ),
+ body: body ? body : undefined,
+ }),
+ next: { revalidate: 0 },
+ signal: signal ?? AbortSignal.timeout(CHECKER_REQUEST_TIMEOUT_MS),
+ });
+ } catch (e) {
+ if (usingDefaultTimeout && isTimeoutError(e)) {
+ throw new TargetUnreachableError("checker request timed out");
+ }
+ throw e;
+ }
const json = await res.json();
const data = checkerSchema.or(errorRequest).safeParse(json);
if (!data.success) {
- console.error(JSON.stringify(res));
- console.error(JSON.stringify(json));
- console.error(
- `something went wrong with request to ${url} error ${data.error.message}`,
- );
- throw new Error(data.error.message);
+ // Neither the success nor the error shape matched — likely checker schema
+ // drift rather than an unreachable target. Warn (not error) so it stays
+ // observable in production without paging Sentry.
+ console.warn("Unexpected checker response shape:", json);
+ throw new TargetUnreachableError(data.error.message);
}
return {
diff --git a/packages/api/src/router/checker.test.ts b/packages/api/src/router/checker.test.ts
index df87db090..4ab6e3afa 100644
--- a/packages/api/src/router/checker.test.ts
+++ b/packages/api/src/router/checker.test.ts
@@ -1,7 +1,7 @@
import { expect } from "@std/expect";
import { afterEach, describe, test } from "@std/testing/bdd";
-import { testGrpc } from "./checker";
+import { testGrpc, testHttp, testTcp } from "./checker";
const originalFetch = globalThis.fetch;
@@ -20,6 +20,117 @@ function stubChecker(body: unknown) {
)) as typeof globalThis.fetch;
}
+/** Make the checker fetch reject, e.g. with the AbortSignal.timeout error. */
+function stubCheckerRejects(error: unknown) {
+ globalThis.fetch = (() => Promise.reject(error)) as typeof globalThis.fetch;
+}
+
+/** Count console.error calls while `fn` runs, then restore. */
+async function countErrors(fn: () => Promise<unknown>): Promise<number> {
+ const origError = console.error;
+ let count = 0;
+ console.error = () => {
+ count++;
+ };
+ try {
+ await fn().catch(() => {});
+ } finally {
+ console.error = origError;
+ }
+ return count;
+}
+
+const timing = {
+ dnsStart: 1,
+ dnsDone: 2,
+ connectStart: 2,
+ connectDone: 3,
+ tlsHandshakeStart: 3,
+ tlsHandshakeDone: 4,
+ firstByteStart: 4,
+ firstByteDone: 5,
+ transferStart: 5,
+ transferDone: 6,
+};
+
+describe("testHttp", () => {
+ const input = {
+ url: "https://example.com",
+ method: "GET" as const,
+ region: "ams" as const,
+ assertions: [],
+ };
+
+ test("accepts a 2XX response", async () => {
+ stubChecker({
+ status: 200,
+ latency: 5,
+ headers: {},
+ timestamp: 1_700_000_000_000,
+ timing,
+ region: "ams",
+ });
+
+ const result = await testHttp(input);
+ expect(result.state).toBe("success");
+ });
+
+ test("maps the unreachable shape to its error message without logging", async () => {
+ // `checker.Http` answers 200 with `error` set and status/headers omitted.
+ stubChecker({
+ error: "Timeout after 45000 ms",
+ latency: 45000,
+ timestamp: 1_700_000_000_000,
+ timing,
+ region: "ams",
+ });
+
+ await expect(testHttp(input)).rejects.toThrow("Timeout after 45000 ms");
+ expect(await countErrors(() => testHttp(input))).toBe(0);
+ });
+
+ test("does not log a failed assertion", async () => {
+ stubChecker({
+ status: 500,
+ latency: 5,
+ headers: {},
+ timestamp: 1_700_000_000_000,
+ timing,
+ region: "ams",
+ });
+
+ await expect(testHttp(input)).rejects.toThrow("Assertion error");
+ expect(await countErrors(() => testHttp(input))).toBe(0);
+ });
+
+ test("maps a checker timeout to BAD_REQUEST without logging", async () => {
+ stubCheckerRejects(new DOMException("timed out", "TimeoutError"));
+
+ const error = await testHttp(input).catch((e) => e);
+ expect(error.code).toBe("BAD_REQUEST");
+ expect(await countErrors(() => testHttp(input))).toBe(0);
+ });
+
+ test("still logs an unexpected failure", async () => {
+ stubCheckerRejects(new TypeError("fetch failed"));
+
+ const error = await testHttp(input).catch((e) => e);
+ expect(error.code).toBe("INTERNAL_SERVER_ERROR");
+ expect(await countErrors(() => testHttp(input))).toBe(1);
+ });
+});
+
+describe("testTcp", () => {
+ const input = { url: "example.com:443", region: "ams" as const };
+
+ test("does not log an unreachable target", async () => {
+ stubChecker({ message: "uri not reachable" });
+
+ await expect(testTcp(input)).rejects.toThrow("uri not reachable");
+ expect(await countErrors(() => testTcp(input))).toBe(0);
+ });
+});
+
/**
* The shape GRPCHandlerRegion returns for a completed RPC. `state` is absent —
* the handler never sends it — so grpcOutput prefaults it to "success".
diff --git a/packages/api/src/router/checker.ts b/packages/api/src/router/checker.ts
index b16ca45e2..314755f7e 100644
--- a/packages/api/src/router/checker.ts
+++ b/packages/api/src/router/checker.ts
@@ -37,6 +37,34 @@ const ICMP_TEST_TIMEOUT = 5000;
// Kept under ABORT_TIMEOUT so the checker answers before the fetch gives up.
const GRPC_TEST_TIMEOUT = 5000;
+// Unreachable targets, failed assertions and timeouts are expected outcomes
+// already surfaced to the user as BAD_REQUEST; only unexpected failures should
+// reach the logs (and Sentry).
+function toCheckerError(
+ error: unknown,
+ label: string,
+ fallback: string,
+): TRPCError {
+ if (error instanceof TRPCError) {
+ if (error.code !== "BAD_REQUEST") {
+ console.error(`Checker ${label} test failed`, error);
+ }
+ return error;
+ }
+
+ if (error instanceof Error && error.name === "TimeoutError") {
+ return new TRPCError({
+ code: "BAD_REQUEST",
+ message: `The ${label} check did not complete within ${
+ ABORT_TIMEOUT / 1000
+ } seconds. Please try again.`,
+ });
+ }
+
+ console.error(`Checker ${label} test failed`, error);
+ return new TRPCError({ code: "INTERNAL_SERVER_ERROR", message: fallback });
+}
+
// Input schemas
const httpTestInput = z.object({
url: safeUrlSchema,
@@ -217,6 +245,13 @@ export const httpOutput = z
state: z.literal("error").prefault("error"),
message: z.string(),
}),
+ )
+ .or(
+ // A target the checker could not reach (timeout, DNS, refused): `checker.Http`
+ // answers 200 with `error` set and `status`/`headers` omitted.
+ z
+ .object({ error: z.string().min(1), timestamp: z.number() })
+ .transform(({ error }) => ({ state: "error" as const, message: error })),
);
export const dnsOutput = z
@@ -325,15 +360,11 @@ export async function testHttp(input: z.infer<typeof httpTestInput>) {
return result.data;
} catch (error) {
- console.error("Checker HTTP test failed", error);
- if (error instanceof TRPCError) {
- throw error;
- }
-
- throw new TRPCError({
- code: "INTERNAL_SERVER_ERROR",
- message: error instanceof Error ? error.message : "HTTP check failed",
- });
+ throw toCheckerError(
+ error,
+ "HTTP",
+ error instanceof Error ? error.message : "HTTP check failed",
+ );
}
}
@@ -376,15 +407,7 @@ export async function testTcp(input: z.infer<typeof tcpTestInput>) {
return result.data;
} catch (error) {
- console.error("Checker TCP test failed", error);
- if (error instanceof TRPCError) {
- throw error;
- }
-
- throw new TRPCError({
- code: "INTERNAL_SERVER_ERROR",
- message: "TCP check failed",
- });
+ throw toCheckerError(error, "TCP", "TCP check failed");
}
}
@@ -446,15 +469,7 @@ export async function testDns(input: z.infer<typeof dnsTestInput>) {
return result.data;
} catch (error) {
- console.error("Checker DNS test failed", error);
- if (error instanceof TRPCError) {
- throw error;
- }
-
- throw new TRPCError({
- code: "INTERNAL_SERVER_ERROR",
- message: "DNS check failed",
- });
+ throw toCheckerError(error, "DNS", "DNS check failed");
}
}
@@ -500,15 +515,7 @@ export async function testIcmp(input: z.infer<typeof icmpTestInput>) {
return result.data;
} catch (error) {
- console.error("Checker ICMP test failed", error);
- if (error instanceof TRPCError) {
- throw error;
- }
-
- throw new TRPCError({
- code: "INTERNAL_SERVER_ERROR",
- message: "ICMP check failed",
- });
+ throw toCheckerError(error, "ICMP", "ICMP check failed");
}
}
@@ -575,15 +582,7 @@ export async function testGrpc(input: z.infer<typeof grpcTestInput>) {
return result.data;
} catch (error) {
- console.error("Checker gRPC test failed", error);
- if (error instanceof TRPCError) {
- throw error;
- }
-
- throw new TRPCError({
- code: "INTERNAL_SERVER_ERROR",
- message: "gRPC check failed",
- });
+ throw toCheckerError(error, "gRPC", "gRPC check failed");
}
}