Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
fix: test checker timeout (#2209)
Maximilian Kaske06bf24d821d1
diff --git a/apps/web/src/app/(landing)/play/checker/[slug]/page.tsx b/apps/web/src/app/(landing)/play/checker/[slug]/page.tsxindex 65c06f078..c54031652 100644--- a/apps/web/src/app/(landing)/play/checker/[slug]/page.tsx+++ b/apps/web/src/app/(landing)/play/checker/[slug]/page.tsx@@ -52,18 +52,22 @@ export async function generateMetadata({if (!data) return metadata;- const regions = data.checks.sort((a, b) => a.latency - b.latency);+ const regions = [...data.checks].sort((a, b) => a.latency - b.latency);const fastestRegion = regions[0];const slowestRegion = regions[regions.length - 1];const TITLE = data.url;- const DESCRIPTION = `${formatDate(- new Date(data.timestamp),- )} | Fastest: ${regionFormatter(fastestRegion.region)} (${latencyFormatter(- fastestRegion.latency,- )}) | Slowest: ${regionFormatter(slowestRegion.region)} (${latencyFormatter(- slowestRegion.latency,- )})`;+ // `checks` is empty when every region failed (only the base hash is stored).+ const DESCRIPTION =+ fastestRegion && slowestRegion+ ? `${formatDate(+ new Date(data.timestamp),+ )} | Fastest: ${regionFormatter(fastestRegion.region)} (${latencyFormatter(+ fastestRegion.latency,+ )}) | Slowest: ${regionFormatter(slowestRegion.region)} (${latencyFormatter(+ slowestRegion.latency,+ )})`+ : `${formatDate(new Date(data.timestamp))} | No successful checks`;return {...metadata,diff --git a/apps/web/src/app/(landing)/play/checker/client.tsx b/apps/web/src/app/(landing)/play/checker/client.tsxindex 1db7ed7cc..5061c6327 100644--- a/apps/web/src/app/(landing)/play/checker/client.tsx+++ b/apps/web/src/app/(landing)/play/checker/client.tsx@@ -132,6 +132,8 @@ export function Form({startTransition(async () => {async function fetchAndReadStream() {let toastId: string | number | undefined;+ let resultId: string | null = null;+ let successCount = 0;try {toastId = toast.loading("Loading data from regions...", {duration: Number.POSITIVE_INFINITY,@@ -171,7 +173,14 @@ export function Form({clearTimeout(timeoutId);const reader = response?.body?.getReader();- if (!reader) return;+ if (!reader) {+ toast.error("Failed to read response", {+ id: toastId,+ description: "Please try again.",+ className: "text-destructive!",+ });+ return;+ }const decoder = new TextDecoder();let done = false;@@ -191,15 +200,7 @@ export function Form({// Store the ID if it's a 32-char hex stringif (is32CharHex(item)) {setId(item);- toast.success("Data is available!", {- id: toastId,- description: "Learn about the response details.",- action: {- label: "Details",- onClick: () => router.push(`/play/checker/${item}`),- },- duration: 4000,- });+ resultId = item;return null;}@@ -225,6 +226,7 @@ export function Form({.filter(notEmpty);if (results.length > 0) {+ successCount += results.length;setValues((prev) => [...prev, ...results]);toast.loading(`Checking ${regionFormatter(@@ -238,13 +240,41 @@ export function Form({}}}++ if (successCount === 0) {+ toast.error("No region could reach the target", {+ id: toastId,+ description: "It may be down or blocking our requests.",+ className: "text-destructive!",+ });+ } else {+ toast.success("Data is available!", {+ id: toastId,+ description: "Learn about the response details.",+ ...(resultId+ ? {+ action: {+ label: "Details",+ onClick: () => router.push(`/play/checker/${resultId}`),+ },+ }+ : {}),+ duration: 4000,+ });+ }} catch (error) {console.error("Error fetching data:", error);if (error instanceof Error && error.name === "AbortError") {toast.error("Request timeout", {id: toastId,description:- "The request took too long and was aborted after 7 seconds.",+ "The request took too long and was aborted after 10 seconds.",+ className: "text-destructive!",+ });+ } else {+ toast.error("Something went wrong", {+ id: toastId,+ description: "Please try again.",className: "text-destructive!",});}diff --git a/apps/web/src/app/api/checker/test/http/route.ts b/apps/web/src/app/api/checker/test/http/route.tsindex 653c0c84f..0edf26944 100644--- a/apps/web/src/app/api/checker/test/http/route.ts+++ b/apps/web/src/app/api/checker/test/http/route.ts@@ -3,7 +3,10 @@ import { httpPayloadSchema } from "@openstatus/utils";import { NextResponse } from "next/server";import { z } from "zod";-import { checkRegion } from "../../../../../lib/checker/utils";+import {+ TargetUnreachableError,+ checkRegion,+} from "../../../../../lib/checker/utils";import { isAnInvalidTestUrl } from "../../utils";export const runtime = "edge";@@ -37,7 +40,10 @@ export async function POST(request: Request) {return NextResponse.json(res);} catch (e) {- console.error(e);+ // Unreachable/timeout targets are expected; only real bugs should reach Sentry.+ if (!(e instanceof TargetUnreachableError)) {+ console.error(e);+ }return NextResponse.json({ success: false }, { status: 400 });}}diff --git a/apps/web/src/app/api/checker/test/tcp/route.ts b/apps/web/src/app/api/checker/test/tcp/route.tsindex 11f89c1dd..800cbdcdf 100644--- a/apps/web/src/app/api/checker/test/tcp/route.ts+++ b/apps/web/src/app/api/checker/test/tcp/route.ts@@ -5,6 +5,12 @@ import {import { NextResponse } from "next/server";import { z } from "zod";+import {+ CHECKER_REQUEST_TIMEOUT_MS,+ TargetUnreachableError,+ isTimeoutError,+} from "@/lib/checker/utils";+import { TCPResponse, tcpPayload } from "./schema";export const runtime = "edge";@@ -34,36 +40,47 @@ export async function POST(request: Request) {return NextResponse.json(res);} catch (e) {- console.error(e);+ // Unreachable/timeout targets are expected; only real bugs should reach Sentry.+ if (!(e instanceof TargetUnreachableError)) {+ console.error(e);+ }return NextResponse.json({ success: false }, { status: 400 });}}async function checkTCP(url: string, region: Region) {//- const res = await fetch(`https://checker.openstatus.dev/tcp/${region}`, {- headers: {- Authorization: `Basic ${process.env.CRON_SECRET}`,- "Content-Type": "application/json",- "fly-prefer-region": region,- },- method: "POST",- body: JSON.stringify({- uri: url,- }),- next: { revalidate: 0 },- });+ let res: Response;+ try {+ res = await fetch(`https://checker.openstatus.dev/tcp/${region}`, {+ headers: {+ Authorization: `Basic ${process.env.CRON_SECRET}`,+ "Content-Type": "application/json",+ "fly-prefer-region": region,+ },+ method: "POST",+ body: JSON.stringify({+ uri: url,+ }),+ next: { revalidate: 0 },+ signal: AbortSignal.timeout(CHECKER_REQUEST_TIMEOUT_MS),+ });+ } catch (e) {+ if (isTimeoutError(e)) {+ throw new TargetUnreachableError("checker request timed out");+ }+ throw e;+ }const json = await res.json();const data = TCPResponse.safeParse(json);+ // A timeout / unreachable target is an expected outcome, not a bug — throw so+ // the caller returns 400, but the catch keeps it out of Sentry. A parse miss+ // here may also be checker schema drift, so warn so it stays observable.if (!data.success) {- console.error(res);- console.error(JSON.stringify(json));- console.error(- `something went wrong with request to ${url} error ${data.error.message}`,- );- throw new Error(data.error.message);+ console.warn("Unexpected TCP checker response shape:", json);+ throw new TargetUnreachableError(data.error.message);}return data.data;diff --git a/apps/web/src/lib/checker/utils.ts b/apps/web/src/lib/checker/utils.tsindex 72ca2ff74..ffe605602 100644--- a/apps/web/src/lib/checker/utils.ts+++ b/apps/web/src/lib/checker/utils.ts@@ -182,6 +182,18 @@ type CheckRegionRequest = {// API Functions// ============================================================================+// A timeout / unreachable target is an expected outcome, not a bug. Callers+// throw this so a route catch can return 400 while keeping it out of Sentry.+export class TargetUnreachableError extends Error {}++// Bound the upstream checker request below the client's 10s abort so a stalled+// checker can't keep the route running to the platform limit.+export const CHECKER_REQUEST_TIMEOUT_MS = 9_000;++export function isTimeoutError(e: unknown): boolean {+ return e instanceof Error && e.name === "TimeoutError";+}+export async function checkRegion(props: CheckRegionRequest,): Promise<RegionCheckerResponse> {@@ -209,40 +221,51 @@ export async function checkRegion(break;}- const res = await fetch(endpoint, {- headers: {- Authorization: `Basic ${process.env.CRON_SECRET}`,- "Content-Type": "application/json",- ...regionHeader,- },- method: "POST",- body: JSON.stringify({- url,- method: method || "GET",- headers: headers?.reduce(- (acc, { key, value }) => {- if (!key) return acc; // key === "" is an invalid header- return { ...acc, [key]: value };- },- {} as Record<string, string>,- ),- body: body ? body : undefined,- }),- signal,- next: { revalidate: 0 },- });+ // A caller-supplied signal is the caller's to interpret (it inspects the raw+ // TimeoutError); only our own default bound maps to TargetUnreachableError.+ const usingDefaultTimeout = !signal;++ let res: Response;+ try {+ res = await fetch(endpoint, {+ headers: {+ Authorization: `Basic ${process.env.CRON_SECRET}`,+ "Content-Type": "application/json",+ ...regionHeader,+ },+ method: "POST",+ body: JSON.stringify({+ url,+ method: method || "GET",+ headers: headers?.reduce(+ (acc, { key, value }) => {+ if (!key) return acc; // key === "" is an invalid header+ return { ...acc, [key]: value };+ },+ {} as Record<string, string>,+ ),+ body: body ? body : undefined,+ }),+ next: { revalidate: 0 },+ signal: signal ?? AbortSignal.timeout(CHECKER_REQUEST_TIMEOUT_MS),+ });+ } catch (e) {+ if (usingDefaultTimeout && isTimeoutError(e)) {+ throw new TargetUnreachableError("checker request timed out");+ }+ throw e;+ }const json = await res.json();const data = checkerSchema.or(errorRequest).safeParse(json);if (!data.success) {- console.error(JSON.stringify(res));- console.error(JSON.stringify(json));- console.error(- `something went wrong with request to ${url} error ${data.error.message}`,- );- throw new Error(data.error.message);+ // Neither the success nor the error shape matched — likely checker schema+ // drift rather than an unreachable target. Warn (not error) so it stays+ // observable in production without paging Sentry.+ console.warn("Unexpected checker response shape:", json);+ throw new TargetUnreachableError(data.error.message);}return {diff --git a/packages/api/src/router/checker.test.ts b/packages/api/src/router/checker.test.tsindex df87db090..4ab6e3afa 100644--- a/packages/api/src/router/checker.test.ts+++ b/packages/api/src/router/checker.test.ts@@ -1,7 +1,7 @@import { expect } from "@std/expect";import { afterEach, describe, test } from "@std/testing/bdd";-import { testGrpc } from "./checker";+import { testGrpc, testHttp, testTcp } from "./checker";const originalFetch = globalThis.fetch;@@ -20,6 +20,117 @@ function stubChecker(body: unknown) {)) as typeof globalThis.fetch;}+/** Make the checker fetch reject, e.g. with the AbortSignal.timeout error. */+function stubCheckerRejects(error: unknown) {+ globalThis.fetch = (() => Promise.reject(error)) as typeof globalThis.fetch;+}++/** Count console.error calls while `fn` runs, then restore. */+async function countErrors(fn: () => Promise<unknown>): Promise<number> {+ const origError = console.error;+ let count = 0;+ console.error = () => {+ count++;+ };+ try {+ await fn().catch(() => {});+ } finally {+ console.error = origError;+ }+ return count;+}++const timing = {+ dnsStart: 1,+ dnsDone: 2,+ connectStart: 2,+ connectDone: 3,+ tlsHandshakeStart: 3,+ tlsHandshakeDone: 4,+ firstByteStart: 4,+ firstByteDone: 5,+ transferStart: 5,+ transferDone: 6,+};++describe("testHttp", () => {+ const input = {+ url: "https://example.com",+ method: "GET" as const,+ region: "ams" as const,+ assertions: [],+ };++ test("accepts a 2XX response", async () => {+ stubChecker({+ status: 200,+ latency: 5,+ headers: {},+ timestamp: 1_700_000_000_000,+ timing,+ region: "ams",+ });++ const result = await testHttp(input);+ expect(result.state).toBe("success");+ });++ test("maps the unreachable shape to its error message without logging", async () => {+ // `checker.Http` answers 200 with `error` set and status/headers omitted.+ stubChecker({+ error: "Timeout after 45000 ms",+ latency: 45000,+ timestamp: 1_700_000_000_000,+ timing,+ region: "ams",+ });++ await expect(testHttp(input)).rejects.toThrow("Timeout after 45000 ms");+ expect(await countErrors(() => testHttp(input))).toBe(0);+ });++ test("does not log a failed assertion", async () => {+ stubChecker({+ status: 500,+ latency: 5,+ headers: {},+ timestamp: 1_700_000_000_000,+ timing,+ region: "ams",+ });++ await expect(testHttp(input)).rejects.toThrow("Assertion error");+ expect(await countErrors(() => testHttp(input))).toBe(0);+ });++ test("maps a checker timeout to BAD_REQUEST without logging", async () => {+ stubCheckerRejects(new DOMException("timed out", "TimeoutError"));++ const error = await testHttp(input).catch((e) => e);+ expect(error.code).toBe("BAD_REQUEST");+ expect(await countErrors(() => testHttp(input))).toBe(0);+ });++ test("still logs an unexpected failure", async () => {+ stubCheckerRejects(new TypeError("fetch failed"));++ const error = await testHttp(input).catch((e) => e);+ expect(error.code).toBe("INTERNAL_SERVER_ERROR");+ expect(await countErrors(() => testHttp(input))).toBe(1);+ });+});++describe("testTcp", () => {+ const input = { url: "example.com:443", region: "ams" as const };++ test("does not log an unreachable target", async () => {+ stubChecker({ message: "uri not reachable" });++ await expect(testTcp(input)).rejects.toThrow("uri not reachable");+ expect(await countErrors(() => testTcp(input))).toBe(0);+ });+});+/*** The shape GRPCHandlerRegion returns for a completed RPC. `state` is absent —* the handler never sends it — so grpcOutput prefaults it to "success".diff --git a/packages/api/src/router/checker.ts b/packages/api/src/router/checker.tsindex b16ca45e2..314755f7e 100644--- a/packages/api/src/router/checker.ts+++ b/packages/api/src/router/checker.ts@@ -37,6 +37,34 @@ const ICMP_TEST_TIMEOUT = 5000;// Kept under ABORT_TIMEOUT so the checker answers before the fetch gives up.const GRPC_TEST_TIMEOUT = 5000;+// Unreachable targets, failed assertions and timeouts are expected outcomes+// already surfaced to the user as BAD_REQUEST; only unexpected failures should+// reach the logs (and Sentry).+function toCheckerError(+ error: unknown,+ label: string,+ fallback: string,+): TRPCError {+ if (error instanceof TRPCError) {+ if (error.code !== "BAD_REQUEST") {+ console.error(`Checker ${label} test failed`, error);+ }+ return error;+ }++ if (error instanceof Error && error.name === "TimeoutError") {+ return new TRPCError({+ code: "BAD_REQUEST",+ message: `The ${label} check did not complete within ${+ ABORT_TIMEOUT / 1000+ } seconds. Please try again.`,+ });+ }++ console.error(`Checker ${label} test failed`, error);+ return new TRPCError({ code: "INTERNAL_SERVER_ERROR", message: fallback });+}+// Input schemasconst httpTestInput = z.object({url: safeUrlSchema,@@ -217,6 +245,13 @@ export const httpOutput = zstate: z.literal("error").prefault("error"),message: z.string(),}),+ )+ .or(+ // A target the checker could not reach (timeout, DNS, refused): `checker.Http`+ // answers 200 with `error` set and `status`/`headers` omitted.+ z+ .object({ error: z.string().min(1), timestamp: z.number() })+ .transform(({ error }) => ({ state: "error" as const, message: error })),);export const dnsOutput = z@@ -325,15 +360,11 @@ export async function testHttp(input: z.infer<typeof httpTestInput>) {return result.data;} catch (error) {- console.error("Checker HTTP test failed", error);- if (error instanceof TRPCError) {- throw error;- }-- throw new TRPCError({- code: "INTERNAL_SERVER_ERROR",- message: error instanceof Error ? error.message : "HTTP check failed",- });+ throw toCheckerError(+ error,+ "HTTP",+ error instanceof Error ? error.message : "HTTP check failed",+ );}}@@ -376,15 +407,7 @@ export async function testTcp(input: z.infer<typeof tcpTestInput>) {return result.data;} catch (error) {- console.error("Checker TCP test failed", error);- if (error instanceof TRPCError) {- throw error;- }-- throw new TRPCError({- code: "INTERNAL_SERVER_ERROR",- message: "TCP check failed",- });+ throw toCheckerError(error, "TCP", "TCP check failed");}}@@ -446,15 +469,7 @@ export async function testDns(input: z.infer<typeof dnsTestInput>) {return result.data;} catch (error) {- console.error("Checker DNS test failed", error);- if (error instanceof TRPCError) {- throw error;- }-- throw new TRPCError({- code: "INTERNAL_SERVER_ERROR",- message: "DNS check failed",- });+ throw toCheckerError(error, "DNS", "DNS check failed");}}@@ -500,15 +515,7 @@ export async function testIcmp(input: z.infer<typeof icmpTestInput>) {return result.data;} catch (error) {- console.error("Checker ICMP test failed", error);- if (error instanceof TRPCError) {- throw error;- }-- throw new TRPCError({- code: "INTERNAL_SERVER_ERROR",- message: "ICMP check failed",- });+ throw toCheckerError(error, "ICMP", "ICMP check failed");}}@@ -575,15 +582,7 @@ export async function testGrpc(input: z.infer<typeof grpcTestInput>) {return result.data;} catch (error) {- console.error("Checker gRPC test failed", error);- if (error instanceof TRPCError) {- throw error;- }-- throw new TRPCError({- code: "INTERNAL_SERVER_ERROR",- message: "gRPC check failed",- });+ throw toCheckerError(error, "gRPC", "gRPC check failed");}}