Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
π notification (#506)
Thibault Le Ouay5e2c577d0c7e
diff --git a/apps/server/src/checker/checker.test.ts b/apps/server/src/checker/checker.test.tsdeleted file mode 100644index a8cf93e97..000000000--- a/apps/server/src/checker/checker.test.ts+++ /dev/null@@ -1,142 +0,0 @@-import { expect, mock, test } from "bun:test";--import { checkerRetryPolicy } from "./checker";--mock.module("./ping.ts", () => {- return {- publishPing: () => {},- };-});--test("should call upsertMonitorStatus when we can fetch", async () => {- const fn = mock(() => {});-- mock.module("./monitor-handler.ts", () => {- return {- handleMonitorFailed: mock(() => {}),- handleMonitorRecovered: fn,- };- });-- await checkerRetryPolicy({- workspaceId: "1",- monitorId: "1",- url: "https://www.google.com",- cronTimestamp: 1,- status: "error",- pageIds: [],- method: "GET",- });- expect(fn).toHaveBeenCalledTimes(1);-});--test("should call upsertMonitorStatus when status error", async () => {- const fn = mock(() => {});-- mock.module("./monitor-handler.ts", () => {- return {- handleMonitorFailed: fn,- handleMonitorRecovered: mock(() => {}),- };- });- try {- await checkerRetryPolicy({- workspaceId: "1",- monitorId: "1",- url: "https://xxxxxxx.fake",- cronTimestamp: 1,- status: "active",- pageIds: [],- method: "GET",- });- } catch (e) {- expect(e).toBeInstanceOf(Error);- }- expect(fn).toHaveBeenCalledTimes(1);-});--test("What should we do when redirect ", async () => {- const fn = mock(() => {});-- mock.module("./monitor-handler.ts", () => {- return {- handleMonitorFailed: fn,- };- });- try {- await checkerRetryPolicy({- workspaceId: "1",- monitorId: "1",- url: "https://www.openstatus.dev/toto",- cronTimestamp: 1,- status: "active",- pageIds: [],- method: "GET",- });- } catch (e) {- expect(e).toBeInstanceOf(Error);- }- expect(fn).toHaveBeenCalledTimes(0);-});--test("When 404 we should trigger alerting ", async () => {- const fn = mock(() => {});- const fn1 = mock(() => {});-- mock.module("./ping.ts", () => {- return {- publishPing: fn,- };- });- mock.module("./monitor-handler.ts", () => {- return {- handleMonitorFailed: fn1,- };- });- try {- await checkerRetryPolicy({- workspaceId: "1",- monitorId: "1",- url: "https://www.openstat.us/404",- cronTimestamp: 1,- status: "active",- pageIds: [],- method: "GET",- });- } catch (e) {- expect(e).toBeInstanceOf(Error);- }- expect(fn).toHaveBeenCalledTimes(1);- expect(fn1).toHaveBeenCalledTimes(1);-});--test("When error 404 we should not trigger alerting ", async () => {- const fn = mock(() => {});- const fn1 = mock(() => {});-- mock.module("./ping.ts", () => {- return {- publishPing: fn,- };- });- mock.module("./monitor-handler.ts", () => {- return {- handleMonitorFailed: fn1,- };- });- try {- await checkerRetryPolicy({- workspaceId: "1",- monitorId: "1",- url: "https://www.openstat.us/404",- cronTimestamp: 1,- status: "error",- pageIds: [],- method: "GET",- });- } catch (e) {- expect(e).toBeInstanceOf(Error);- }- expect(fn).toHaveBeenCalledTimes(1);- expect(fn1).toHaveBeenCalledTimes(0);-});diff --git a/apps/server/src/checker/checker.ts b/apps/server/src/checker/checker.tsdeleted file mode 100644index 151116208..000000000--- a/apps/server/src/checker/checker.ts+++ /dev/null@@ -1,136 +0,0 @@-import { handleMonitorFailed, handleMonitorRecovered } from "./monitor-handler";-import type { PublishPingType } from "./ping";-import { getHeaders, publishPing } from "./ping";-import type { Payload } from "./schema";--// we could have a 'retry' parameter to know how often we should retry-// we could use a setTimeout to retry after a certain amount of time - can be random between 500ms and 10s-export const publishPingRetryPolicy = async ({- payload,- latency,- statusCode,- message,-}: PublishPingType) => {- try {- console.log(- `1οΈβ£ try publish ping to tb - attempt 1 ${JSON.stringify(- payload,- )} with latency ${latency} and status code ${statusCode}`,- );- await publishPing({- payload,- statusCode,- latency,- message,- });- } catch {- try {- console.log(- "2οΈβ£ try publish ping to tb - attempt 2 ",- JSON.stringify(payload),- );-- await publishPing({- payload,- statusCode,- latency,- message,- });- } catch (e) {- throw e;- }- }- console.log(- `ποΈ Successfully published ${JSON.stringify(- payload,- )} with latency ${latency} and status code ${statusCode}`,- );-};--const run = async (data: Payload, retry: number) => {- let startTime = 0;- let endTime = 0;- let res = null;- let message = undefined;- // We are doing these for wrong urls- try {- const headers = getHeaders(data);- console.log(`π fetch is about to start for ${JSON.stringify(data)}`);- startTime = performance.now();- res = await fetch(data.url, {- method: data.method,- keepalive: false,- cache: "no-store",- headers,- // Avoid having "TypeError: Request with a GET or HEAD method cannot have a body." error- ...(data.method === "POST" && { body: data?.body }),- });-- endTime = performance.now();- console.log(`β fetch is done for ${JSON.stringify(data)}`);- } catch (e) {- endTime = performance.now();- message = `${e}`;- console.log(- `π¨ error on pingEndpoint for ${JSON.stringify(data)} error: `,- e,- );- }-- const latency = Number((endTime - startTime).toFixed(0));- if (res?.ok) {- await publishPingRetryPolicy({- payload: data,- latency,- statusCode: res.status,- message: undefined,- });- if (data?.status === "error") {- await handleMonitorRecovered(data, res);- }- } else {- if (retry < 2) {- throw new Error(`error on ping for ${data.monitorId}`);- }- // Store the error on third task retry- if (retry === 2) {- console.log(- `π error on fetching data for ${JSON.stringify(- data,- )} with result ${JSON.stringify(res)}`,- );- await publishPingRetryPolicy({- payload: data,- latency,- statusCode: res?.status,- message,- });- if (data.status === "active") {- await handleMonitorFailed(data, res, message);- }- }- }- return { res, latency };-};--// We have this extensible retry policy because sometimes we have this error :-// ConnectionClosed: The socket connection was closed unexpectedly.-export const checkerRetryPolicy = async (data: Payload) => {- try {- console.log("π₯ try run checker - attempt 1 ", JSON.stringify(data));- await run(data, 0);- } catch {- try {- console.log("π₯ try run checker - attempt 2 ", JSON.stringify(data));- await run(data, 1);- } catch (e) {- try {- console.log("π₯ try run checker - attempt 3 ", JSON.stringify(data));- await run(data, 2);- } catch (e) {- throw e;- }- }- }- console.log("π₯ successfully run checker ", JSON.stringify(data));-};diff --git a/apps/server/src/checker/index.ts b/apps/server/src/checker/index.tsindex af3d50c86..0de6bd207 100644--- a/apps/server/src/checker/index.ts+++ b/apps/server/src/checker/index.ts@@ -5,119 +5,9 @@ import { flyRegions } from "@openstatus/db/src/schema/monitors/constants";import { env } from "../env";import { checkerAudit } from "../utils/audit-log";-import { upsertMonitorStatus } from "./alerting";-import { checkerRetryPolicy } from "./checker";-import { payloadSchema } from "./schema";-import type { Payload } from "./schema";+import { triggerAlerting, upsertMonitorStatus } from "./alerting";-export type Variables = {- payload: Payload;-};--export const checkerRoute = new Hono<{ Variables: Variables }>();--// TODO: only use checkerRoute.post("/checker", checker);--checkerRoute.post("/checker", async (c) => {- const json = await c.req.json();- const auth = c.req.header("Authorization");- if (auth !== `Basic ${env.CRON_SECRET}`) {- console.error("Unauthorized");- return c.text("Unauthorized", 401);- }- console.log("Retry : ", c.req.header("X-CloudTasks-TaskRetryCount"));-- const result = payloadSchema.safeParse(json);-- if (!result.success) {- console.error(result.error);- return c.text("Unprocessable Entity", 422);- }- const retry = Number(c.req.header("X-CloudTasks-TaskRetryCount") || 0);- if (retry > 3) {- console.error(- `catchTooManyRetry for ${JSON.stringify(result.data)}- )}`,- );- return c.text("Ok", 200); // finish the task- }-- try {- console.log(- `start checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,- );- await checkerRetryPolicy(result.data);- console.log(- `end checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,- );- return c.text("Ok", 200);- } catch (e) {- console.error(- `fail checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,- JSON.stringify(result.data),- e,- );- if (result.data.status === "error") {- console.log(- `The monitor was already in error we should not retry checker URL: ${result.data}`,- );- return c.text("Ok", 200);- }- return c.text("Internal Server Error", 500);- }-});--checkerRoute.post("/checkerV2", async (c) => {- const json = await c.req.json();-- const auth = c.req.header("Authorization");- if (auth !== `Basic ${env.CRON_SECRET}`) {- console.error("Unauthorized");- return c.text("Unauthorized", 401);- }-- const result = payloadSchema.safeParse(json);-- if (!result.success) {- console.error(result.error);- return c.text("Unprocessable Entity", 422);- }- const retry = Number(c.req.header("X-CloudTasks-TaskRetryCount") || 0);- if (retry > 2) {- console.error(- `β Too many retry for ${JSON.stringify(result.data)}- )}`,- );- // catchTooManyRetry(result.data);- return c.text("Ok", 200); // finish the task- }- if (retry > 0 && result.data.status === "error") {- console.log(- `ποΈ The monitor was already in error we should not checked the URL: ${result.data}`,- );- return c.text("Ok", 200);- }-- try {- console.log(`π§ start checker for: ${JSON.stringify(result.data)}`);- await checkerRetryPolicy(result.data);- console.log(`π end checker for: ${JSON.stringify(result.data)} `);- return c.text("Ok", 200);- } catch (e) {- if (result.data.status === "error") {- console.log(- `ποΈ The monitor was already in error we should not retry checker URL: ${result.data}`,- );- return c.text("Ok", 200);- }- console.error(- `π΄ fail checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,- JSON.stringify(result.data),- e,- );- return c.text("Internal Server Error", 500);- }-});+export const checkerRoute = new Hono();checkerRoute.post("/updateStatus", async (c) => {const auth = c.req.header("Authorization");@@ -146,14 +36,15 @@ checkerRoute.post("/updateStatus", async (c) => {switch (status) {case "active":+ if (!statusCode) {+ return;+ }await upsertMonitorStatus({monitorId: monitorId,status: "active",region: region,});- if (!statusCode) {- return;- }+await checkerAudit.publishAuditLog({id: `monitor:${monitorId}`,action: "monitor.recovered",@@ -178,6 +69,12 @@ checkerRoute.post("/updateStatus", async (c) => {message,},});+ await triggerAlerting({+ monitorId: monitorId,+ region: env.FLY_REGION,+ statusCode,+ message,+ });break;}return c.text("Ok", 200);diff --git a/apps/server/src/checker/monitor-handler.ts b/apps/server/src/checker/monitor-handler.tsdeleted file mode 100644index 62ad2edec..000000000--- a/apps/server/src/checker/monitor-handler.ts+++ /dev/null@@ -1,48 +0,0 @@-import { env } from "../env";-import { checkerAudit } from "../utils/audit-log";-import { triggerAlerting, upsertMonitorStatus } from "./alerting";-import type { Payload } from "./schema";--export async function handleMonitorRecovered(data: Payload, res: Response) {- // await upsertMonitorStatus({- // monitorId: data.monitorId,- // status: "active",- // });- // ALPHA- await checkerAudit.publishAuditLog({- id: `monitor:${data.monitorId}`,- action: "monitor.recovered",- targets: [{ id: data.monitorId, type: "monitor" }],- metadata: { region: env.FLY_REGION, statusCode: res.status },- });- //-}--export async function handleMonitorFailed(- data: Payload,- res: Response | null,- message?: string,-) {- // await upsertMonitorStatus({- // monitorId: data.monitorId,- // status: "error",- // });- // ALPHA- await checkerAudit.publishAuditLog({- id: `monitor:${data.monitorId}`,- action: "monitor.failed",- targets: [{ id: data.monitorId, type: "monitor" }],- metadata: {- region: env.FLY_REGION,- statusCode: res?.status,- message,- },- });- //- await triggerAlerting({- monitorId: data.monitorId,- region: env.FLY_REGION,- statusCode: res?.status,- message,- });-}diff --git a/apps/server/src/checker/ping.ts b/apps/server/src/checker/ping.tsdeleted file mode 100644index cee9b35de..000000000--- a/apps/server/src/checker/ping.ts+++ /dev/null@@ -1,60 +0,0 @@-import { publishPingResponse } from "@openstatus/tinybird";--import { env } from "../env";-import { fakePromiseWithRandomResolve } from "../utils/random-promise";-import type { Payload } from "./schema";--const region = env.FLY_REGION;--export function getHeaders(data?: Payload) {- const customHeaders =- data?.headers?.reduce((o, v) => {- // removes empty keys from the header- if (v.key.trim() === "") return o;- return { ...o, [v.key]: v.value };- }, {}) || {};- return {- "OpenStatus-Ping": "true",- ...customHeaders,- };-}--export type PublishPingType = {- payload: Payload;- latency: number;- statusCode?: number | undefined;- message?: string | undefined;-};--export async function publishPing({- payload,- latency,- message,- statusCode,-}: PublishPingType) {- const { monitorId, cronTimestamp, url, workspaceId } = payload;-- if (- process.env.NODE_ENV === "production" ||- process.env.NODE_ENV === "test"- ) {- const res = await publishPingResponse({- timestamp: Date.now(),- statusCode,- latency,- region,- url,- monitorId,- cronTimestamp,- workspaceId,- message: message,- });- if (res.successful_rows === 0) {- throw new Error(`error 0 rows on publish ping for ${payload.monitorId}`);- }- return res;- }-- const res = await fakePromiseWithRandomResolve();- return res;-}diff --git a/apps/server/src/checker/schema.ts b/apps/server/src/checker/schema.tsdeleted file mode 100644index 90009a375..000000000--- a/apps/server/src/checker/schema.ts+++ /dev/null@@ -1,17 +0,0 @@-import { z } from "zod";--import { monitorMethods, monitorStatus } from "@openstatus/db/src/schema";--export const payloadSchema = z.object({- workspaceId: z.string(),- monitorId: z.string(),- method: z.enum(monitorMethods),- body: z.string().optional(),- headers: z.array(z.object({ key: z.string(), value: z.string() })).optional(),- url: z.string(),- cronTimestamp: z.number(),- pageIds: z.array(z.string()),- status: z.enum(monitorStatus),-});--export type Payload = z.infer<typeof payloadSchema>;diff --git a/apps/server/src/index.ts b/apps/server/src/index.tsindex 625778d53..37eb67a56 100644--- a/apps/server/src/index.ts+++ b/apps/server/src/index.ts@@ -6,16 +6,10 @@ import { checkerRoute } from "./checker";import { env } from "./env";import { publicRoute } from "./public";import { api } from "./v1";-import { VercelIngest } from "./vercel";const app = new Hono();app.use("*", sentry({ dsn: process.env.SENTRY_DSN }));-/**- * Vercel Integration- */-app.post("/integration/vercel", VercelIngest);-/*** Public Routes*/diff --git a/apps/server/src/schema/vercel.ts b/apps/server/src/schema/vercel.tsdeleted file mode 100644index b1a19b61b..000000000--- a/apps/server/src/schema/vercel.ts+++ /dev/null@@ -1,46 +0,0 @@-import { z } from "zod";--// https://vercel.com/docs/observability/log-drains-overview/log-drains-reference#json-log-drains-export const logDrainSchema = z.object({- id: z.string().optional(),- timestamp: z.number().optional(),- type: z- .enum([- "middleware-invocation",- "stdout",- "stderr",- "edge-function-invocation",- "fatal",- ])- .optional(),- edgeType: z.enum(["edge-function", "middleware"]).optional(),- requestId: z.string().optional(),- statusCode: z.number().optional(),- message: z.string().optional(),- projectId: z.string().optional(),- deploymentId: z.string().optional(),- buildId: z.string().optional(),- source: z.enum(["external", "lambda", "edge", "static", "build"]),- host: z.string().optional(),- environment: z.string().optional(),- branch: z.string().optional(),- destination: z.string().optional(),- path: z.string().optional(),- entrypoint: z.string().optional(),- proxy: z- .object({- timestamp: z.number().optional(),- region: z.string().optional(), // TODO: use regions enum?- method: z.string().optional(), // TODO: use methods enum?- vercelCache: z.string().optional(), // TODO: use "HIT" / "MISS" enum?- statusCode: z.number().optional(),- path: z.string().optional(),- host: z.string().optional(),- scheme: z.string().optional(),- clientIp: z.string().optional(),- userAgent: z.array(z.string()).optional(),- })- .optional(),-});--export const logDrainSchemaArray = z.array(logDrainSchema);diff --git a/apps/server/src/vercel.ts b/apps/server/src/vercel.tsdeleted file mode 100644index 92e9ab1a9..000000000--- a/apps/server/src/vercel.ts+++ /dev/null@@ -1,36 +0,0 @@-import type { Context, Env } from "hono";--import { Tinybird } from "@openstatus/tinybird";--import { env } from "./env";-import { logDrainSchema, logDrainSchemaArray } from "./schema/vercel";--const tb = new Tinybird({ token: env.TINY_BIRD_API_KEY }); // should we use t3-env?--export function publishVercelLogDrain() {- return tb.buildIngestEndpoint({- datasource: "vercel_log_drain__v0",- event: logDrainSchema,- });-}--export const VercelIngest = async (- c: Context<Env, "/integration/vercel", {}>,-) => {- const json = c.req.json();- const logDrains = logDrainSchemaArray.safeParse(json);-- if (logDrains.success) {- // We are only pushing the logs that are not stdout or stderr- const data = logDrains.data.filter(- (log) => log.type !== "stdout" && log.type !== "stderr",- );-- for (const event of data) {- // FIXME: Zod-bird is broken- await publishVercelLogDrain()(event);- }- }-- c.json({ code: "ok" });-};