Skip to content

Open-source status page, incident response and uptime monitoring

18.6 KB
Raw Blame
import { Events } from "@openstatus/analytics";
import {
  deserialize,
  dnsRecords,
  headerAssertion,
  jsonBodyAssertion,
  recordAssertion,
  statusAssertion,
  textBodyAssertion,
} from "@openstatus/assertions";
import { and, db, eq } from "@openstatus/db";
import { monitor, selectMonitorSchema } from "@openstatus/db/src/schema";
import { monitorRegionSchema } from "@openstatus/db/src/schema/constants";
import {
  GRPC_TLS_MODES,
  buildCheckerPayload,
  headerPairSchema,
  safeUrlSchema,
  transformHeaders,
} from "@openstatus/utils";
import { TRPCError } from "@trpc/server";
import { z } from "zod";

import { env } from "../env";
import { createTRPCRouter, protectedProcedure } from "../trpc";

const ABORT_TIMEOUT = 10000;
const CHECKER_BASE_URL = (
  env.CHECKER_URL || "https://openstatus-checker.fly.dev"
).replace(/\/+$/, "");

// PingICMP treats its timeout as the deadline for the whole check, so omitting
// it means a deadline of "now": the send loop breaks before the first packet
// and every test reports "no reply". Kept under ABORT_TIMEOUT so the checker
// answers before the fetch above gives up.
const ICMP_TEST_TIMEOUT = 5000;

// Kept under ABORT_TIMEOUT so the checker answers before the fetch gives up.
const GRPC_TEST_TIMEOUT = 5000;

// Unreachable targets, failed assertions and timeouts are expected outcomes
// already surfaced to the user as BAD_REQUEST; only unexpected failures should
// reach the logs (and Sentry).
function toCheckerError(
  error: unknown,
  label: string,
  fallback: string,
): TRPCError {
  if (error instanceof TRPCError) {
    if (error.code !== "BAD_REQUEST") {
      console.error(`Checker ${label} test failed`, error);
    }
    return error;
  }

  if (error instanceof Error && error.name === "TimeoutError") {
    return new TRPCError({
      code: "BAD_REQUEST",
      message: `The ${label} check did not complete within ${
        ABORT_TIMEOUT / 1000
      } seconds. Please try again.`,
    });
  }

  console.error(`Checker ${label} test failed`, error);
  return new TRPCError({ code: "INTERNAL_SERVER_ERROR", message: fallback });
}

// Input schemas
const httpTestInput = z.object({
  url: safeUrlSchema,
  method: z
    .enum([
      "GET",
      "HEAD",
      "OPTIONS",
      "POST",
      "PUT",
      "DELETE",
      "PATCH",
      "CONNECT",
      "TRACE",
    ])
    .prefault("GET"),
  headers: z.array(headerPairSchema).optional(),
  body: z.string().optional(),
  region: monitorRegionSchema.prefault("ams"),
  assertions: z
    .array(
      z.discriminatedUnion("type", [
        statusAssertion,
        headerAssertion,
        textBodyAssertion,
        jsonBodyAssertion,
        recordAssertion,
      ]),
    )
    .prefault([]),
});

const tcpTestInput = z.object({
  url: z.string(),
  region: monitorRegionSchema.prefault("ams"),
});

const dnsTestInput = z.object({
  url: z.string(),
  region: monitorRegionSchema.prefault("ams"),
  assertions: z
    .array(
      z.discriminatedUnion("type", [
        recordAssertion,
        statusAssertion,
        headerAssertion,
        textBodyAssertion,
        jsonBodyAssertion,
      ]),
    )
    .prefault([]),
});

const icmpTestInput = z.object({
  url: z.string(),
  region: monitorRegionSchema.prefault("ams"),
});

const grpcTestInput = z.object({
  url: z.string(),
  service: z.string().optional(),
  tls: z.enum(GRPC_TLS_MODES).prefault("tls"),
  headers: z.array(headerPairSchema).optional(),
  region: monitorRegionSchema.prefault("ams"),
});

export const grpcOutput = z
  .object({
    state: z.literal("success").prefault("success"),
    type: z.literal("grpc").prefault("grpc"),
    jobType: z.literal("grpc").optional(),
    requestId: z.number().optional(),
    workspaceId: z.number().optional(),
    monitorId: z.number().optional(),
    timestamp: z.number(),
    timing: z.object({
      dnsStart: z.number(),
      dnsDone: z.number(),
      connectStart: z.number(),
      connectDone: z.number(),
      tlsHandshakeStart: z.number(),
      tlsHandshakeDone: z.number(),
      firstByteStart: z.number(),
      firstByteDone: z.number(),
      transferStart: z.number(),
      transferDone: z.number(),
    }),
    latency: z.number().optional(),
    servingStatus: z.string().optional(),
    service: z.string().optional(),
    grpcCode: z.number().optional(),
    completed: z.boolean().optional(),
    errorMessage: z.string().optional(),
    error: z.number().optional(),
    region: monitorRegionSchema,
  })
  .or(
    z.object({
      state: z.literal("error").prefault("error"),
      message: z.string(),
    }),
  );

export const icmpOutput = z
  .object({
    state: z.literal("success").prefault("success"),
    type: z.literal("icmp").prefault("icmp"),
    requestId: z.number().optional(),
    workspaceId: z.number().optional(),
    monitorId: z.number().optional(),
    timestamp: z.number(),
    timing: z.object({
      rtts: z.array(z.number()),
    }),
    latency: z.number().optional(),
    latencyMin: z.number().optional(),
    latencyMax: z.number().optional(),
    packetsSent: z.number().optional(),
    packetsReceived: z.number().optional(),
    error: z.string().optional(),
    region: monitorRegionSchema,
  })
  .or(
    z.object({
      state: z.literal("error").prefault("error"),
      message: z.string(),
    }),
  );

export const tcpOutput = z
  .object({
    state: z.literal("success").prefault("success"),
    type: z.literal("tcp").prefault("tcp"),
    requestId: z.number().optional(),
    workspaceId: z.number().optional(),
    monitorId: z.number().optional(),
    timestamp: z.number(),
    timing: z.object({
      tcpStart: z.number(),
      tcpDone: z.number(),
    }),
    error: z.string().optional(),
    region: monitorRegionSchema,
    latency: z.number().optional(),
  })
  .or(
    z.object({
      state: z.literal("error").prefault("error"),
      message: z.string(),
    }),
  );

export const httpOutput = z
  .object({
    state: z.literal("success").prefault("success"),
    type: z.literal("http").prefault("http"),
    status: z.number(),
    latency: z.number(),
    headers: z.record(z.string(), z.string()),
    timestamp: z.number(),
    timing: z.object({
      dnsStart: z.number(),
      dnsDone: z.number(),
      connectStart: z.number(),
      connectDone: z.number(),
      tlsHandshakeStart: z.number(),
      tlsHandshakeDone: z.number(),
      firstByteStart: z.number(),
      firstByteDone: z.number(),
      transferStart: z.number(),
      transferDone: z.number(),
    }),
    body: z.string().optional().nullable(),
    region: monitorRegionSchema,
  })
  .or(
    z.object({
      state: z.literal("error").prefault("error"),
      message: z.string(),
    }),
  )
  .or(
    // A target the checker could not reach (timeout, DNS, refused): `checker.Http`
    // answers 200 with `error` set and `status`/`headers` omitted.
    z
      .object({ error: z.string().min(1), timestamp: z.number() })
      .transform(({ error }) => ({ state: "error" as const, message: error })),
  );

export const dnsOutput = z
  .object({
    state: z.literal("success").prefault("success"),
    type: z.literal("dns").prefault("dns"),
    records: z
      .partialRecord(z.enum(dnsRecords), z.array(z.string()))
      .prefault({}),
    latency: z.number().optional(),
    timestamp: z.number(),
    region: monitorRegionSchema,
  })
  .or(
    z.object({
      state: z.literal("error").prefault("error"),
      message: z.string(),
    }),
  );

export async function testHttp(input: z.infer<typeof httpTestInput>) {
  // Reject requests to our own domain to avoid loops
  if (input.url.includes("openstatus.dev")) {
    throw new TRPCError({
      code: "BAD_REQUEST",
      message: "Self-requests are not allowed",
    });
  }

  try {
    const res = await fetch(`${CHECKER_BASE_URL}/ping/${input.region}`, {
      method: "POST",
      headers: {
        Authorization: `Basic ${env.CRON_SECRET}`,
        "Content-Type": "application/json",
        "fly-prefer-region": input.region,
      },
      body: JSON.stringify({
        url: input.url,
        method: input.method,
        headers: input.headers?.reduce(
          (acc, { key, value }) => {
            if (!key) return acc;
            return { ...acc, [key]: value };
          },
          {} as Record<string, string>,
        ),
        body: input.body,
      }),
      signal: AbortSignal.timeout(ABORT_TIMEOUT),
    });

    const json = await res.json();
    const result = httpOutput.safeParse(json);

    if (!result.success) {
      console.error(
        `Checker HTTP test failed for ${input.url}:`,
        result.error.message,
      );
      throw new TRPCError({
        code: "BAD_REQUEST",
        message:
          "Checker response is not valid. Please try again. If the problem persists, please contact support.",
      });
    }

    if (result.data.state === "error") {
      throw new TRPCError({
        code: "BAD_REQUEST",
        message: result.data.message,
      });
    }

    if (result.data.state === "success") {
      const { body, headers, status } = result.data;

      const assertions = deserialize(JSON.stringify(input.assertions)).map(
        (assertion) =>
          assertion.assert({
            body: body ?? "",
            header: headers ?? {},
            status: status,
          }),
      );

      if (assertions.some((assertion) => !assertion.success)) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message: `Assertion error: ${
            assertions.find((assertion) => !assertion.success)?.message
          }`,
        });
      }

      if (assertions.length === 0 && (status < 200 || status >= 300)) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message: `Assertion error: The response status was not 2XX: ${status}.`,
        });
      }
    }

    return result.data;
  } catch (error) {
    throw toCheckerError(
      error,
      "HTTP",
      error instanceof Error ? error.message : "HTTP check failed",
    );
  }
}

export async function testTcp(input: z.infer<typeof tcpTestInput>) {
  try {
    const res = await fetch(`${CHECKER_BASE_URL}/tcp/${input.region}`, {
      method: "POST",
      headers: {
        Authorization: `Basic ${env.CRON_SECRET}`,
        "Content-Type": "application/json",
        "fly-prefer-region": input.region,
      },
      body: JSON.stringify({ uri: input.url }),
      signal: AbortSignal.timeout(ABORT_TIMEOUT),
    });

    const json = await res.json();
    const result = tcpOutput.safeParse(json);

    if (!result.success) {
      console.error(
        `Checker TCP test failed for ${input.url}:`,
        result.error.message,
      );
      throw new TRPCError({
        code: "BAD_REQUEST",
        message: `Checker response is not valid. Please try again. If the problem persists, please contact support. ${result.error.message}`,
      });
    }

    if (result.data.state === "error") {
      throw new TRPCError({
        code: "BAD_REQUEST",
        message: result.data.message,
      });
    }

    return result.data;
  } catch (error) {
    throw toCheckerError(error, "TCP", "TCP check failed");
  }
}

export async function testDns(input: z.infer<typeof dnsTestInput>) {
  try {
    const res = await fetch(`${CHECKER_BASE_URL}/dns/${input.region}`, {
      method: "POST",
      headers: {
        Authorization: `Basic ${env.CRON_SECRET}`,
        "Content-Type": "application/json",
        "fly-prefer-region": input.region,
      },
      body: JSON.stringify({
        uri: input.url,
      }),
      signal: AbortSignal.timeout(ABORT_TIMEOUT),
    });

    const json = await res.json();
    const result = dnsOutput.safeParse(json);

    if (!result.success) {
      console.error(
        `Checker DNS test failed for ${input.url}:`,
        result.error.message,
      );
      throw new TRPCError({
        code: "BAD_REQUEST",
        message: `Checker response is not valid. Please try again. If the problem persists, please contact support. ${result.error.message}`,
      });
    }

    if (result.data.state === "error") {
      throw new TRPCError({
        code: "BAD_REQUEST",
        message: result.data.message,
      });
    }

    if (result.data.state === "success") {
      const { records } = result.data;

      const assertions = deserialize(JSON.stringify(input.assertions)).map(
        (assertion) => assertion.assert({ records }),
      );

      if (assertions.some((assertion) => !assertion.success)) {
        throw new TRPCError({
          code: "BAD_REQUEST",
          message: `Assertion error: ${
            assertions.find((assertion) => !assertion.success)?.message
          }`,
        });
      }
    }

    return result.data;
  } catch (error) {
    throw toCheckerError(error, "DNS", "DNS check failed");
  }
}

export async function testIcmp(input: z.infer<typeof icmpTestInput>) {
  try {
    const res = await fetch(`${CHECKER_BASE_URL}/icmp/${input.region}`, {
      method: "POST",
      headers: {
        Authorization: `Basic ${env.CRON_SECRET}`,
        "Content-Type": "application/json",
        "fly-prefer-region": input.region,
      },
      body: JSON.stringify({
        uri: input.url,
        timeout: ICMP_TEST_TIMEOUT,
      }),
      signal: AbortSignal.timeout(ABORT_TIMEOUT),
    });

    const json = await res.json();
    const result = icmpOutput.safeParse(json);

    if (!result.success) {
      console.error(
        `Checker ICMP test failed for ${input.url}:`,
        result.error.message,
      );
      throw new TRPCError({
        code: "BAD_REQUEST",
        message: `Checker response is not valid. Please try again. If the problem persists, please contact support. ${result.error.message}`,
      });
    }

    if (result.data.state === "error") {
      throw new TRPCError({
        code: "BAD_REQUEST",
        message: result.data.message,
      });
    }

    return result.data;
  } catch (error) {
    throw toCheckerError(error, "ICMP", "ICMP check failed");
  }
}

export async function testGrpc(input: z.infer<typeof grpcTestInput>) {
  try {
    const res = await fetch(`${CHECKER_BASE_URL}/grpc/${input.region}`, {
      method: "POST",
      headers: {
        Authorization: `Basic ${env.CRON_SECRET}`,
        "Content-Type": "application/json",
        "fly-prefer-region": input.region,
      },
      body: JSON.stringify({
        uri: input.url,
        service: input.service,
        tls: input.tls,
        headers: transformHeaders(input.headers ?? []),
        timeout: GRPC_TEST_TIMEOUT,
      }),
      signal: AbortSignal.timeout(ABORT_TIMEOUT),
    });

    const json = await res.json();
    const result = grpcOutput.safeParse(json);

    if (!result.success) {
      console.error(
        `Checker gRPC test failed for ${input.url}:`,
        result.error.message,
      );
      throw new TRPCError({
        code: "BAD_REQUEST",
        message: `Checker response is not valid. Please try again. If the problem persists, please contact support. ${result.error.message}`,
      });
    }

    if (result.data.state === "error") {
      throw new TRPCError({
        code: "BAD_REQUEST",
        message: result.data.message,
      });
    }

    // Only a transport failure comes back as `state: "error"`. An RPC that
    // completed but answered NOT_SERVING / SERVICE_UNKNOWN — or a server with no
    // health service at all — returns the full response, where `state` is absent
    // and prefaults to "success". `error` is omitempty, so it is present only
    // when the check failed. Mirrors testHttp rejecting a non-2XX status: the
    // target is reachable, but saving it would create a monitor that is already
    // down.
    if (result.data.error === 1) {
      throw new TRPCError({
        code: "BAD_REQUEST",
        message:
          result.data.errorMessage ||
          `The health check did not report SERVING${
            result.data.servingStatus ? `: ${result.data.servingStatus}` : ""
          }`,
      });
    }

    return result.data;
  } catch (error) {
    throw toCheckerError(error, "gRPC", "gRPC check failed");
  }
}

export async function triggerChecker(
  input: z.infer<typeof selectMonitorSchema>,
) {
  if (process.env.NODE_ENV !== "production") {
    return;
  }

  const payload = buildCheckerPayload({
    monitor: input,
    status: "active",
    trigger: "cron",
    timestamp: Date.now(),
  });
  const allResult = [];

  for (const region of input.regions) {
    const res = fetch(generateUrl({ row: input }), {
      method: "POST",
      headers: {
        Authorization: `Basic ${env.CRON_SECRET}`,
        "Content-Type": "application/json",
        "fly-prefer-region": region,
      },
      body: JSON.stringify(payload),
      signal: AbortSignal.timeout(ABORT_TIMEOUT),
    });
    allResult.push(res);
  }

  await Promise.allSettled(allResult);
}

function generateUrl({ row }: { row: z.infer<typeof selectMonitorSchema> }) {
  switch (row.jobType) {
    case "http":
      return `${CHECKER_BASE_URL}/checker/http?monitor_id=${row.id}`;
    case "tcp":
      return `${CHECKER_BASE_URL}/checker/tcp?monitor_id=${row.id}`;
    case "dns":
      return `${CHECKER_BASE_URL}/checker/dns?monitor_id=${row.id}`;
    case "icmp":
      return `${CHECKER_BASE_URL}/checker/icmp?monitor_id=${row.id}`;
    case "grpc":
      return `${CHECKER_BASE_URL}/checker/grpc?monitor_id=${row.id}`;
    default:
      throw new Error("Invalid jobType");
  }
}

export const checkerRouter = createTRPCRouter({
  testHttp: protectedProcedure
    .meta({ track: Events.TestMonitor })
    .input(httpTestInput)
    .mutation(async ({ input }) => {
      return testHttp(input);
    }),

  testTcp: protectedProcedure
    .meta({ track: Events.TestMonitor })
    .input(tcpTestInput)
    .mutation(async ({ input }) => {
      return testTcp(input);
    }),
  testDns: protectedProcedure
    .meta({ track: Events.TestMonitor })
    .input(dnsTestInput)
    .mutation(async ({ input }) => {
      return testDns(input);
    }),
  testIcmp: protectedProcedure
    .meta({ track: Events.TestMonitor })
    .input(icmpTestInput)
    .mutation(async ({ input }) => {
      return testIcmp(input);
    }),

  testGrpc: protectedProcedure
    .meta({ track: Events.TestMonitor })
    .input(grpcTestInput)
    .mutation(async ({ input }) => {
      return testGrpc(input);
    }),

  triggerChecker: protectedProcedure
    .input(z.object({ id: z.number() }))
    .mutation(async (opts) => {
      const m = await db
        .select()
        .from(monitor)
        .where(
          and(
            eq(monitor.id, opts.input.id),
            eq(monitor.workspaceId, opts.ctx.workspace.id),
          ),
        )
        .get();
      if (!m) {
        throw new TRPCError({
          code: "NOT_FOUND",
          message: "Monitor not found",
        });
      }
      const input = selectMonitorSchema.parse(m);

      return await triggerChecker(input);
    }),
});