Skip to content

Open-source status page, incident response and uptime monitoring

πŸ› notification (#506)

Thibault Le Ouay5e2c577d0c7e
diff --git a/apps/server/src/checker/checker.test.ts b/apps/server/src/checker/checker.test.ts
deleted file mode 100644
index a8cf93e97..000000000
--- a/apps/server/src/checker/checker.test.ts
+++ /dev/null
@@ -1,142 +0,0 @@
-import { expect, mock, test } from "bun:test";
-
-import { checkerRetryPolicy } from "./checker";
-
-mock.module("./ping.ts", () => {
- return {
- publishPing: () => {},
- };
-});
-
-test("should call upsertMonitorStatus when we can fetch", async () => {
- const fn = mock(() => {});
-
- mock.module("./monitor-handler.ts", () => {
- return {
- handleMonitorFailed: mock(() => {}),
- handleMonitorRecovered: fn,
- };
- });
-
- await checkerRetryPolicy({
- workspaceId: "1",
- monitorId: "1",
- url: "https://www.google.com",
- cronTimestamp: 1,
- status: "error",
- pageIds: [],
- method: "GET",
- });
- expect(fn).toHaveBeenCalledTimes(1);
-});
-
-test("should call upsertMonitorStatus when status error", async () => {
- const fn = mock(() => {});
-
- mock.module("./monitor-handler.ts", () => {
- return {
- handleMonitorFailed: fn,
- handleMonitorRecovered: mock(() => {}),
- };
- });
- try {
- await checkerRetryPolicy({
- workspaceId: "1",
- monitorId: "1",
- url: "https://xxxxxxx.fake",
- cronTimestamp: 1,
- status: "active",
- pageIds: [],
- method: "GET",
- });
- } catch (e) {
- expect(e).toBeInstanceOf(Error);
- }
- expect(fn).toHaveBeenCalledTimes(1);
-});
-
-test("What should we do when redirect ", async () => {
- const fn = mock(() => {});
-
- mock.module("./monitor-handler.ts", () => {
- return {
- handleMonitorFailed: fn,
- };
- });
- try {
- await checkerRetryPolicy({
- workspaceId: "1",
- monitorId: "1",
- url: "https://www.openstatus.dev/toto",
- cronTimestamp: 1,
- status: "active",
- pageIds: [],
- method: "GET",
- });
- } catch (e) {
- expect(e).toBeInstanceOf(Error);
- }
- expect(fn).toHaveBeenCalledTimes(0);
-});
-
-test("When 404 we should trigger alerting ", async () => {
- const fn = mock(() => {});
- const fn1 = mock(() => {});
-
- mock.module("./ping.ts", () => {
- return {
- publishPing: fn,
- };
- });
- mock.module("./monitor-handler.ts", () => {
- return {
- handleMonitorFailed: fn1,
- };
- });
- try {
- await checkerRetryPolicy({
- workspaceId: "1",
- monitorId: "1",
- url: "https://www.openstat.us/404",
- cronTimestamp: 1,
- status: "active",
- pageIds: [],
- method: "GET",
- });
- } catch (e) {
- expect(e).toBeInstanceOf(Error);
- }
- expect(fn).toHaveBeenCalledTimes(1);
- expect(fn1).toHaveBeenCalledTimes(1);
-});
-
-test("When error 404 we should not trigger alerting ", async () => {
- const fn = mock(() => {});
- const fn1 = mock(() => {});
-
- mock.module("./ping.ts", () => {
- return {
- publishPing: fn,
- };
- });
- mock.module("./monitor-handler.ts", () => {
- return {
- handleMonitorFailed: fn1,
- };
- });
- try {
- await checkerRetryPolicy({
- workspaceId: "1",
- monitorId: "1",
- url: "https://www.openstat.us/404",
- cronTimestamp: 1,
- status: "error",
- pageIds: [],
- method: "GET",
- });
- } catch (e) {
- expect(e).toBeInstanceOf(Error);
- }
- expect(fn).toHaveBeenCalledTimes(1);
- expect(fn1).toHaveBeenCalledTimes(0);
-});
diff --git a/apps/server/src/checker/checker.ts b/apps/server/src/checker/checker.ts
deleted file mode 100644
index 151116208..000000000
--- a/apps/server/src/checker/checker.ts
+++ /dev/null
@@ -1,136 +0,0 @@
-import { handleMonitorFailed, handleMonitorRecovered } from "./monitor-handler";
-import type { PublishPingType } from "./ping";
-import { getHeaders, publishPing } from "./ping";
-import type { Payload } from "./schema";
-
-// we could have a 'retry' parameter to know how often we should retry
-// we could use a setTimeout to retry after a certain amount of time - can be random between 500ms and 10s
-export const publishPingRetryPolicy = async ({
- payload,
- latency,
- statusCode,
- message,
-}: PublishPingType) => {
- try {
- console.log(
- `1️⃣ try publish ping to tb - attempt 1 ${JSON.stringify(
- payload,
- )} with latency ${latency} and status code ${statusCode}`,
- );
- await publishPing({
- payload,
- statusCode,
- latency,
- message,
- });
- } catch {
- try {
- console.log(
- "2️⃣ try publish ping to tb - attempt 2 ",
- JSON.stringify(payload),
- );
-
- await publishPing({
- payload,
- statusCode,
- latency,
- message,
- });
- } catch (e) {
- throw e;
- }
- }
- console.log(
- `πŸ—ƒοΈ Successfully published ${JSON.stringify(
- payload,
- )} with latency ${latency} and status code ${statusCode}`,
- );
-};
-
-const run = async (data: Payload, retry: number) => {
- let startTime = 0;
- let endTime = 0;
- let res = null;
- let message = undefined;
- // We are doing these for wrong urls
- try {
- const headers = getHeaders(data);
- console.log(`πŸ†• fetch is about to start for ${JSON.stringify(data)}`);
- startTime = performance.now();
- res = await fetch(data.url, {
- method: data.method,
- keepalive: false,
- cache: "no-store",
- headers,
- // Avoid having "TypeError: Request with a GET or HEAD method cannot have a body." error
- ...(data.method === "POST" && { body: data?.body }),
- });
-
- endTime = performance.now();
- console.log(`βœ… fetch is done for ${JSON.stringify(data)}`);
- } catch (e) {
- endTime = performance.now();
- message = `${e}`;
- console.log(
- `🚨 error on pingEndpoint for ${JSON.stringify(data)} error: `,
- e,
- );
- }
-
- const latency = Number((endTime - startTime).toFixed(0));
- if (res?.ok) {
- await publishPingRetryPolicy({
- payload: data,
- latency,
- statusCode: res.status,
- message: undefined,
- });
- if (data?.status === "error") {
- await handleMonitorRecovered(data, res);
- }
- } else {
- if (retry < 2) {
- throw new Error(`error on ping for ${data.monitorId}`);
- }
- // Store the error on third task retry
- if (retry === 2) {
- console.log(
- `πŸ› error on fetching data for ${JSON.stringify(
- data,
- )} with result ${JSON.stringify(res)}`,
- );
- await publishPingRetryPolicy({
- payload: data,
- latency,
- statusCode: res?.status,
- message,
- });
- if (data.status === "active") {
- await handleMonitorFailed(data, res, message);
- }
- }
- }
- return { res, latency };
-};
-
-// We have this extensible retry policy because sometimes we have this error :
-// ConnectionClosed: The socket connection was closed unexpectedly.
-export const checkerRetryPolicy = async (data: Payload) => {
- try {
- console.log("πŸ₯‡ try run checker - attempt 1 ", JSON.stringify(data));
- await run(data, 0);
- } catch {
- try {
- console.log("πŸ₯ˆ try run checker - attempt 2 ", JSON.stringify(data));
- await run(data, 1);
- } catch (e) {
- try {
- console.log("πŸ₯‰ try run checker - attempt 3 ", JSON.stringify(data));
- await run(data, 2);
- } catch (e) {
- throw e;
- }
- }
- }
- console.log("πŸ”₯ successfully run checker ", JSON.stringify(data));
-};
diff --git a/apps/server/src/checker/index.ts b/apps/server/src/checker/index.ts
index af3d50c86..0de6bd207 100644
--- a/apps/server/src/checker/index.ts
+++ b/apps/server/src/checker/index.ts
@@ -5,119 +5,9 @@ import { flyRegions } from "@openstatus/db/src/schema/monitors/constants";
import { env } from "../env";
import { checkerAudit } from "../utils/audit-log";
-import { upsertMonitorStatus } from "./alerting";
-import { checkerRetryPolicy } from "./checker";
-import { payloadSchema } from "./schema";
-import type { Payload } from "./schema";
+import { triggerAlerting, upsertMonitorStatus } from "./alerting";
-export type Variables = {
- payload: Payload;
-};
-
-export const checkerRoute = new Hono<{ Variables: Variables }>();
-
-// TODO: only use checkerRoute.post("/checker", checker);
-
-checkerRoute.post("/checker", async (c) => {
- const json = await c.req.json();
- const auth = c.req.header("Authorization");
- if (auth !== `Basic ${env.CRON_SECRET}`) {
- console.error("Unauthorized");
- return c.text("Unauthorized", 401);
- }
- console.log("Retry : ", c.req.header("X-CloudTasks-TaskRetryCount"));
-
- const result = payloadSchema.safeParse(json);
-
- if (!result.success) {
- console.error(result.error);
- return c.text("Unprocessable Entity", 422);
- }
- const retry = Number(c.req.header("X-CloudTasks-TaskRetryCount") || 0);
- if (retry > 3) {
- console.error(
- `catchTooManyRetry for ${JSON.stringify(result.data)}
- )}`,
- );
- return c.text("Ok", 200); // finish the task
- }
-
- try {
- console.log(
- `start checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
- );
- await checkerRetryPolicy(result.data);
- console.log(
- `end checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
- );
- return c.text("Ok", 200);
- } catch (e) {
- console.error(
- `fail checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
- JSON.stringify(result.data),
- e,
- );
- if (result.data.status === "error") {
- console.log(
- `The monitor was already in error we should not retry checker URL: ${result.data}`,
- );
- return c.text("Ok", 200);
- }
- return c.text("Internal Server Error", 500);
- }
-});
-
-checkerRoute.post("/checkerV2", async (c) => {
- const json = await c.req.json();
-
- const auth = c.req.header("Authorization");
- if (auth !== `Basic ${env.CRON_SECRET}`) {
- console.error("Unauthorized");
- return c.text("Unauthorized", 401);
- }
-
- const result = payloadSchema.safeParse(json);
-
- if (!result.success) {
- console.error(result.error);
- return c.text("Unprocessable Entity", 422);
- }
- const retry = Number(c.req.header("X-CloudTasks-TaskRetryCount") || 0);
- if (retry > 2) {
- console.error(
- `β›” Too many retry for ${JSON.stringify(result.data)}
- )}`,
- );
- // catchTooManyRetry(result.data);
- return c.text("Ok", 200); // finish the task
- }
- if (retry > 0 && result.data.status === "error") {
- console.log(
- `πŸ—‘οΈ The monitor was already in error we should not checked the URL: ${result.data}`,
- );
- return c.text("Ok", 200);
- }
-
- try {
- console.log(`🧭 start checker for: ${JSON.stringify(result.data)}`);
- await checkerRetryPolicy(result.data);
- console.log(`πŸ”š end checker for: ${JSON.stringify(result.data)} `);
- return c.text("Ok", 200);
- } catch (e) {
- if (result.data.status === "error") {
- console.log(
- `πŸ—‘οΈ The monitor was already in error we should not retry checker URL: ${result.data}`,
- );
- return c.text("Ok", 200);
- }
- console.error(
- `πŸ”΄ fail checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
- JSON.stringify(result.data),
- e,
- );
- return c.text("Internal Server Error", 500);
- }
-});
+export const checkerRoute = new Hono();
checkerRoute.post("/updateStatus", async (c) => {
const auth = c.req.header("Authorization");
@@ -146,14 +36,15 @@ checkerRoute.post("/updateStatus", async (c) => {
switch (status) {
case "active":
+ if (!statusCode) {
+ return;
+ }
await upsertMonitorStatus({
monitorId: monitorId,
status: "active",
region: region,
});
- if (!statusCode) {
- return;
- }
+
await checkerAudit.publishAuditLog({
id: `monitor:${monitorId}`,
action: "monitor.recovered",
@@ -178,6 +69,12 @@ checkerRoute.post("/updateStatus", async (c) => {
message,
},
});
+ await triggerAlerting({
+ monitorId: monitorId,
+ region: env.FLY_REGION,
+ statusCode,
+ message,
+ });
break;
}
return c.text("Ok", 200);
diff --git a/apps/server/src/checker/monitor-handler.ts b/apps/server/src/checker/monitor-handler.ts
deleted file mode 100644
index 62ad2edec..000000000
--- a/apps/server/src/checker/monitor-handler.ts
+++ /dev/null
@@ -1,48 +0,0 @@
-import { env } from "../env";
-import { checkerAudit } from "../utils/audit-log";
-import { triggerAlerting, upsertMonitorStatus } from "./alerting";
-import type { Payload } from "./schema";
-
-export async function handleMonitorRecovered(data: Payload, res: Response) {
- // await upsertMonitorStatus({
- // monitorId: data.monitorId,
- // status: "active",
- // });
- // ALPHA
- await checkerAudit.publishAuditLog({
- id: `monitor:${data.monitorId}`,
- action: "monitor.recovered",
- targets: [{ id: data.monitorId, type: "monitor" }],
- metadata: { region: env.FLY_REGION, statusCode: res.status },
- });
- //
-}
-
-export async function handleMonitorFailed(
- data: Payload,
- res: Response | null,
- message?: string,
-) {
- // await upsertMonitorStatus({
- // monitorId: data.monitorId,
- // status: "error",
- // });
- // ALPHA
- await checkerAudit.publishAuditLog({
- id: `monitor:${data.monitorId}`,
- action: "monitor.failed",
- targets: [{ id: data.monitorId, type: "monitor" }],
- metadata: {
- region: env.FLY_REGION,
- statusCode: res?.status,
- message,
- },
- });
- //
- await triggerAlerting({
- monitorId: data.monitorId,
- region: env.FLY_REGION,
- statusCode: res?.status,
- message,
- });
-}
diff --git a/apps/server/src/checker/ping.ts b/apps/server/src/checker/ping.ts
deleted file mode 100644
index cee9b35de..000000000
--- a/apps/server/src/checker/ping.ts
+++ /dev/null
@@ -1,60 +0,0 @@
-import { publishPingResponse } from "@openstatus/tinybird";
-
-import { env } from "../env";
-import { fakePromiseWithRandomResolve } from "../utils/random-promise";
-import type { Payload } from "./schema";
-
-const region = env.FLY_REGION;
-
-export function getHeaders(data?: Payload) {
- const customHeaders =
- data?.headers?.reduce((o, v) => {
- // removes empty keys from the header
- if (v.key.trim() === "") return o;
- return { ...o, [v.key]: v.value };
- }, {}) || {};
- return {
- "OpenStatus-Ping": "true",
- ...customHeaders,
- };
-}
-
-export type PublishPingType = {
- payload: Payload;
- latency: number;
- statusCode?: number | undefined;
- message?: string | undefined;
-};
-
-export async function publishPing({
- payload,
- latency,
- message,
- statusCode,
-}: PublishPingType) {
- const { monitorId, cronTimestamp, url, workspaceId } = payload;
-
- if (
- process.env.NODE_ENV === "production" ||
- process.env.NODE_ENV === "test"
- ) {
- const res = await publishPingResponse({
- timestamp: Date.now(),
- statusCode,
- latency,
- region,
- url,
- monitorId,
- cronTimestamp,
- workspaceId,
- message: message,
- });
- if (res.successful_rows === 0) {
- throw new Error(`error 0 rows on publish ping for ${payload.monitorId}`);
- }
- return res;
- }
-
- const res = await fakePromiseWithRandomResolve();
- return res;
-}
diff --git a/apps/server/src/checker/schema.ts b/apps/server/src/checker/schema.ts
deleted file mode 100644
index 90009a375..000000000
--- a/apps/server/src/checker/schema.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-import { z } from "zod";
-
-import { monitorMethods, monitorStatus } from "@openstatus/db/src/schema";
-
-export const payloadSchema = z.object({
- workspaceId: z.string(),
- monitorId: z.string(),
- method: z.enum(monitorMethods),
- body: z.string().optional(),
- headers: z.array(z.object({ key: z.string(), value: z.string() })).optional(),
- url: z.string(),
- cronTimestamp: z.number(),
- pageIds: z.array(z.string()),
- status: z.enum(monitorStatus),
-});
-
-export type Payload = z.infer<typeof payloadSchema>;
diff --git a/apps/server/src/index.ts b/apps/server/src/index.ts
index 625778d53..37eb67a56 100644
--- a/apps/server/src/index.ts
+++ b/apps/server/src/index.ts
@@ -6,16 +6,10 @@ import { checkerRoute } from "./checker";
import { env } from "./env";
import { publicRoute } from "./public";
import { api } from "./v1";
-import { VercelIngest } from "./vercel";
const app = new Hono();
app.use("*", sentry({ dsn: process.env.SENTRY_DSN }));
-/**
- * Vercel Integration
- */
-app.post("/integration/vercel", VercelIngest);
-
/**
* Public Routes
*/
diff --git a/apps/server/src/schema/vercel.ts b/apps/server/src/schema/vercel.ts
deleted file mode 100644
index b1a19b61b..000000000
--- a/apps/server/src/schema/vercel.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-import { z } from "zod";
-
-// https://vercel.com/docs/observability/log-drains-overview/log-drains-reference#json-log-drains
-export const logDrainSchema = z.object({
- id: z.string().optional(),
- timestamp: z.number().optional(),
- type: z
- .enum([
- "middleware-invocation",
- "stdout",
- "stderr",
- "edge-function-invocation",
- "fatal",
- ])
- .optional(),
- edgeType: z.enum(["edge-function", "middleware"]).optional(),
- requestId: z.string().optional(),
- statusCode: z.number().optional(),
- message: z.string().optional(),
- projectId: z.string().optional(),
- deploymentId: z.string().optional(),
- buildId: z.string().optional(),
- source: z.enum(["external", "lambda", "edge", "static", "build"]),
- host: z.string().optional(),
- environment: z.string().optional(),
- branch: z.string().optional(),
- destination: z.string().optional(),
- path: z.string().optional(),
- entrypoint: z.string().optional(),
- proxy: z
- .object({
- timestamp: z.number().optional(),
- region: z.string().optional(), // TODO: use regions enum?
- method: z.string().optional(), // TODO: use methods enum?
- vercelCache: z.string().optional(), // TODO: use "HIT" / "MISS" enum?
- statusCode: z.number().optional(),
- path: z.string().optional(),
- host: z.string().optional(),
- scheme: z.string().optional(),
- clientIp: z.string().optional(),
- userAgent: z.array(z.string()).optional(),
- })
- .optional(),
-});
-
-export const logDrainSchemaArray = z.array(logDrainSchema);
diff --git a/apps/server/src/vercel.ts b/apps/server/src/vercel.ts
deleted file mode 100644
index 92e9ab1a9..000000000
--- a/apps/server/src/vercel.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-import type { Context, Env } from "hono";
-
-import { Tinybird } from "@openstatus/tinybird";
-
-import { env } from "./env";
-import { logDrainSchema, logDrainSchemaArray } from "./schema/vercel";
-
-const tb = new Tinybird({ token: env.TINY_BIRD_API_KEY }); // should we use t3-env?
-
-export function publishVercelLogDrain() {
- return tb.buildIngestEndpoint({
- datasource: "vercel_log_drain__v0",
- event: logDrainSchema,
- });
-}
-
-export const VercelIngest = async (
- c: Context<Env, "/integration/vercel", {}>,
-) => {
- const json = c.req.json();
- const logDrains = logDrainSchemaArray.safeParse(json);
-
- if (logDrains.success) {
- // We are only pushing the logs that are not stdout or stderr
- const data = logDrains.data.filter(
- (log) => log.type !== "stdout" && log.type !== "stderr",
- );
-
- for (const event of data) {
- // FIXME: Zod-bird is broken
- await publishVercelLogDrain()(event);
- }
- }
-
- c.json({ code: "ok" });
-};