Skip to content

Open-source status page, incident response and uptime monitoring

🥅 improve error (#404)

Thibault Le Ouaya21f46fc487a
diff --git a/apps/server/.env.example b/apps/server/.env.example
index 89dc76823..90f7308a5 100644
--- a/apps/server/.env.example
+++ b/apps/server/.env.example
@@ -4,4 +4,5 @@ NODE_ENV=production
UNKEY_TOKEN=
TINY_BIRD_API_KEY=
UPSTASH_REDIS_REST_URL=
-UPSTASH_REDIS_REST_TOKEN=
\ No newline at end of file
+UPSTASH_REDIS_REST_TOKEN=
+RESEND_API_KEY=
\ No newline at end of file
diff --git a/apps/server/fly.toml b/apps/server/fly.toml
index 885dbc5e1..fe3151fc8 100644
--- a/apps/server/fly.toml
+++ b/apps/server/fly.toml
@@ -19,7 +19,7 @@ primary_region = "ams"
processes = ["app"]
[deploy]
- strategy = "bluegreen"
+ strategy = "rolling"
[[http_service.checks]]
grace_period = "10s"
diff --git a/apps/server/src/checker/alerting.ts b/apps/server/src/checker/alerting.ts
index df78199e2..4238b828d 100644
--- a/apps/server/src/checker/alerting.ts
+++ b/apps/server/src/checker/alerting.ts
@@ -3,12 +3,12 @@ import type { z } from "zod";
import { db, eq, schema } from "@openstatus/db";
import { selectNotificationSchema } from "@openstatus/db/src/schema";
-import { monitor } from "./checker";
+import { publishPingRetryPolicy } from "./checker";
import type { Payload } from "./schema";
import { providerToFunction } from "./utils";
export async function catchTooManyRetry(payload: Payload) {
- await monitor({ monitorInfo: payload, latency: -1, statusCode: 500 });
+ await publishPingRetryPolicy({ payload, latency: -1, statusCode: 500 });
if (payload?.status !== "error") {
await triggerAlerting({ monitorId: payload.monitorId });
await updateMonitorStatus({
diff --git a/apps/server/src/checker/checker.test.ts b/apps/server/src/checker/checker.test.ts
index d4f92e0d6..3f173955d 100644
--- a/apps/server/src/checker/checker.test.ts
+++ b/apps/server/src/checker/checker.test.ts
@@ -8,20 +8,20 @@ import {
} from "@openstatus/tinybird";
import * as alerts from "./alerting";
-import { checker } from "./checker";
+import { checkerRetryPolicy } from "./checker";
vi.mock("@openstatus/tinybird", async () => {
const actual = await vi.importActual("@openstatus/tinybird");
return {
// @ts-ignore
...actual,
- publishPingResponse: vi.fn(),
+ publishPingResponse: vi.fn().mockResolvedValue({ successful_rows: 1 }),
};
});
it("should call updateMonitorStatus when we can fetch", async () => {
const spyOn = vi.spyOn(alerts, "updateMonitorStatus").mockReturnThis();
- await checker({
+ await checkerRetryPolicy({
workspaceId: "1",
monitorId: "1",
url: "https://www.google.com",
@@ -36,7 +36,7 @@ it("should call updateMonitorStatus when we can fetch", async () => {
it("should call updateMonitorStatus when status error", async () => {
const spyOn = vi.spyOn(alerts, "updateMonitorStatus").mockReturnThis();
try {
- await checker({
+ await checkerRetryPolicy({
workspaceId: "1",
monitorId: "1",
url: "https://xxxxxxx.fake",
@@ -48,13 +48,14 @@ it("should call updateMonitorStatus when status error", async () => {
} catch (e) {
expect(e).toBeInstanceOf(Error);
}
- expect(spyOn).toHaveBeenCalledTimes(0);
+ expect(spyOn).toHaveBeenCalledTimes(1);
});
it("What should we do when redirect ", async () => {
const spyOn = vi.spyOn(alerts, "updateMonitorStatus").mockReturnThis();
+
try {
- await checker({
+ await checkerRetryPolicy({
workspaceId: "1",
monitorId: "1",
url: "https://www.openstatus.dev/toto",
diff --git a/apps/server/src/checker/checker.ts b/apps/server/src/checker/checker.ts
index 04aed6890..aab413440 100644
--- a/apps/server/src/checker/checker.ts
+++ b/apps/server/src/checker/checker.ts
@@ -1,47 +1,61 @@
-import { nanoid } from "nanoid";
-
-import { publishPingResponse } from "@openstatus/tinybird";
-
-import { env } from "../env";
import { updateMonitorStatus } from "./alerting";
+import type { PublishPingType } from "./ping";
+import { pingEndpoint, publishPing } from "./ping";
import type { Payload } from "./schema";
-const region = env.FLY_REGION;
-
-export const monitor = async ({
- monitorInfo,
+// we could have a 'retry' parameter to know how often we should retry
+// we could use a setTimeout to retry after a certain amount of time - can be random between 500ms and 10s
+export const publishPingRetryPolicy = async ({
+ payload,
latency,
statusCode,
-}: {
- monitorInfo: Payload;
- latency: number;
- statusCode: number;
-}) => {
- const { monitorId, cronTimestamp, url, workspaceId } = monitorInfo;
-
+}: PublishPingType) => {
+ try {
+ console.log(
+ `try publish ping to tb - attempt 1 ${JSON.stringify(
+ payload,
+ )} with latency ${latency} and status code ${statusCode}`,
+ );
+ await publishPing({ payload, statusCode, latency });
+ } catch {
+ try {
+ console.log(
+ "try publish ping to tb - attempt 2 ",
+ JSON.stringify(payload),
+ );
+ await publishPing({ payload, statusCode, latency });
+ } catch (e) {
+ throw e;
+ }
+ }
console.log(
- `publishing ping response for ${url} with status ${statusCode} and latency ${latency} and monitorId ${monitorId} `,
+ `Successfully published ${JSON.stringify(
+ payload,
+ )} with latency ${latency} and status code ${statusCode}`,
);
- await publishPingResponse({
- id: nanoid(), // TBD: we don't need it
- timestamp: Date.now(),
- statusCode,
- latency,
- region,
- url,
- monitorId,
- cronTimestamp,
- workspaceId,
- });
};
-export const checker = async (data: Payload) => {
- const startTime = Date.now();
- const res = await ping(data);
- const endTime = Date.now();
+const run = async (data: Payload, retry?: number | undefined) => {
+ let startTime = 0;
+ let endTime = 0;
+ let res = null;
+ // We are doing these for wrong urls
+ try {
+ startTime = Date.now();
+ res = await pingEndpoint(data);
+ endTime = Date.now();
+ } catch (e) {
+ console.log("error on pingEndpoint", e);
+ endTime = Date.now();
+ }
+
const latency = endTime - startTime;
if (res?.ok) {
- await monitor({ monitorInfo: data, latency, statusCode: res.status });
+ await publishPingRetryPolicy({
+ payload: data,
+ latency,
+ statusCode: res.status,
+ });
if (data?.status === "error") {
await updateMonitorStatus({
monitorId: data.monitorId,
@@ -49,51 +63,38 @@ export const checker = async (data: Payload) => {
});
}
} else {
- console.log(`first retry for ${data.url} with status ${res?.status}`);
- const startTime = Date.now();
- const retry = await ping(data);
- const endTime = Date.now();
- const latency = endTime - startTime;
- if (retry?.ok) {
- await monitor({ monitorInfo: data, latency, statusCode: retry.status });
- if (data?.status === "error") {
+ if (retry === 0) {
+ throw new Error(`error on ping for ${data.monitorId}`);
+ }
+ // Store the error on third task retry
+ if (retry === 1) {
+ await publishPingRetryPolicy({
+ payload: data,
+ latency,
+ statusCode: res?.status || 0,
+ });
+ if (data?.status === "active") {
await updateMonitorStatus({
monitorId: data.monitorId,
- status: "active",
+ status: "error",
});
}
- } else {
- console.log(
- `error for ${JSON.stringify(data)} with info ${JSON.stringify(retry)}`,
- );
}
}
+ return { res, latency };
};
-export const ping = async (
- data: Pick<Payload, "headers" | "body" | "method" | "url">,
-) => {
- const headers =
- data?.headers?.reduce((o, v) => {
- if (v.key.trim() === "") return o; // removes empty keys from the header
- return { ...o, [v.key]: v.value };
- }, {}) || {};
-
+export const checkerRetryPolicy = async (data: Payload, retry = 0) => {
try {
- const res = await fetch(data?.url, {
- method: data?.method,
- cache: "no-store",
- headers: {
- "OpenStatus-Ping": "true",
- ...headers,
- },
- // Avoid having "TypeError: Request with a GET or HEAD method cannot have a body." error
- ...(data.method === "POST" && { body: data?.body }),
- });
-
- return res;
- } catch (e) {
- console.log(`fetch error for : ${data} with error ${e}`);
- console.log(e);
+ console.log("try run checker - attempt 1 ", JSON.stringify(data));
+ await run(data, 0);
+ } catch {
+ try {
+ console.log("try run checker - attempt 2 ", JSON.stringify(data));
+ await run(data, 1);
+ } catch (e) {
+ throw e;
+ }
}
+ console.log("successfully run checker ", JSON.stringify(data));
};
diff --git a/apps/server/src/checker/index.ts b/apps/server/src/checker/index.ts
index 99e59309d..6eae274c2 100644
--- a/apps/server/src/checker/index.ts
+++ b/apps/server/src/checker/index.ts
@@ -1,8 +1,7 @@
import { Hono } from "hono";
import { env } from "../env";
-import { catchTooManyRetry } from "./alerting";
-import { checker } from "./checker";
+import { checkerRetryPolicy } from "./checker";
import { payloadSchema } from "./schema";
import type { Payload } from "./schema";
@@ -30,14 +29,14 @@ checkerRoute.post("/checker", async (c) => {
console.error(result.error);
return c.text("Unprocessable Entity", 422);
}
-
- if (Number(c.req.header("X-CloudTasks-TaskRetryCount") || 0) > 5) {
+ const retry = Number(c.req.header("X-CloudTasks-TaskRetryCount") || 0);
+ if (retry > 3) {
console.error(
`catchTooManyRetry for ${JSON.stringify(result.data)}
)}`,
);
// catchTooManyRetry(result.data);
- return c.text("Ok", 200); // needs to be 200, otherwise qstash will retry
+ return c.text("Ok", 200); // finish the task
}
console.log(`Google Checker should try this: ${result.data.url}`);
@@ -46,13 +45,17 @@ checkerRoute.post("/checker", async (c) => {
console.log(
`start checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
);
- checker(result.data);
+ await checkerRetryPolicy(result.data, retry);
console.log(
`end checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
);
return c.text("Ok", 200);
} catch (e) {
- console.error(e);
+ console.error(
+ `fail checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
+ JSON.stringify(result.data),
+ e,
+ );
return c.text("Internal Server Error", 500);
}
});
@@ -73,14 +76,14 @@ checkerRoute.post("/checkerV2", async (c) => {
console.error(result.error);
return c.text("Unprocessable Entity", 422);
}
-
- if (Number(c.req.header("X-CloudTasks-TaskRetryCount") || 0) > 5) {
+ const retry = Number(c.req.header("X-CloudTasks-TaskRetryCount") || 0);
+ if (retry > 1) {
console.error(
`catchTooManyRetry for ${JSON.stringify(result.data)}
)}`,
);
// catchTooManyRetry(result.data);
- return c.text("Ok", 200); // needs to be 200, otherwise qstash will retry
+ return c.text("Ok", 200); // finish the task
}
console.log(`Google Checker should try this: ${result.data.url}`);
@@ -89,13 +92,17 @@ checkerRoute.post("/checkerV2", async (c) => {
console.log(
`start checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
);
- checker(result.data);
+ await checkerRetryPolicy(result.data, retry);
console.log(
`end checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
);
return c.text("Ok", 200);
} catch (e) {
- console.error(e);
+ console.error(
+ `fail checker URL: ${result.data.url} monitorId ${result.data.monitorId}`,
+ JSON.stringify(result.data),
+ e,
+ );
return c.text("Internal Server Error", 500);
}
});
diff --git a/apps/server/src/checker/ping.ts b/apps/server/src/checker/ping.ts
new file mode 100644
index 000000000..6bfe5e318
--- /dev/null
+++ b/apps/server/src/checker/ping.ts
@@ -0,0 +1,77 @@
+import { nanoid } from "nanoid";
+
+import { publishPingResponse } from "@openstatus/tinybird";
+
+import { env } from "../env";
+import { fakePromiseWithRandomResolve } from "../utils/random-promise";
+import type { Payload } from "./schema";
+
+const region = env.FLY_REGION;
+
+function getHeaders(data?: Payload) {
+ const customHeaders =
+ data?.headers?.reduce((o, v) => {
+ // removes empty keys from the header
+ if (v.key.trim() === "") return o;
+ return { ...o, [v.key]: v.value };
+ }, {}) || {};
+ return {
+ "OpenStatus-Ping": "true",
+ ...customHeaders,
+ };
+}
+
+export async function pingEndpoint(data: Payload) {
+ try {
+ const res = await fetch(data?.url, {
+ method: data?.method,
+ keepalive: false,
+ cache: "no-store",
+ headers: getHeaders(data),
+ // Avoid having "TypeError: Request with a GET or HEAD method cannot have a body." error
+ ...(data.method === "POST" && { body: data?.body }),
+ });
+
+ return res;
+ } catch (e) {
+ throw e;
+ }
+}
+
+export type PublishPingType = {
+ payload: Payload;
+ statusCode: number;
+ latency: number;
+};
+
+export async function publishPing({
+ payload,
+ statusCode,
+ latency,
+}: PublishPingType) {
+ const { monitorId, cronTimestamp, url, workspaceId } = payload;
+
+ if (
+ process.env.NODE_ENV === "production" ||
+ process.env.NODE_ENV === "test"
+ ) {
+ const res = await publishPingResponse({
+ id: nanoid(), // TBD: we don't need it
+ timestamp: Date.now(),
+ statusCode,
+ latency,
+ region,
+ url,
+ monitorId,
+ cronTimestamp,
+ workspaceId,
+ });
+ if (res.successful_rows === 0) {
+ throw new Error(`error 0 rows on publish ping for ${payload.monitorId}`);
+ }
+ return res;
+ }
+
+ const res = await fakePromiseWithRandomResolve();
+ return res;
+}
diff --git a/apps/server/src/utils/random-promise.ts b/apps/server/src/utils/random-promise.ts
new file mode 100644
index 000000000..229328af5
--- /dev/null
+++ b/apps/server/src/utils/random-promise.ts
@@ -0,0 +1,13 @@
+export function fakePromiseWithRandomResolve() {
+ return new Promise((resolve, reject) => {
+ const randomTime = Math.floor(Math.random() * 1000);
+ setTimeout(() => {
+ const shouldResolve = Math.random() < 0; // 0.5
+ if (shouldResolve) {
+ resolve("Promise resolved successfully.");
+ } else {
+ reject(new Error("Promise rejected."));
+ }
+ }, randomTime);
+ });
+}
diff --git a/apps/web/src/app/api/checker/cron/_cron.ts b/apps/web/src/app/api/checker/cron/_cron.ts
index 8132d4d56..7e330ee7f 100644
--- a/apps/web/src/app/api/checker/cron/_cron.ts
+++ b/apps/web/src/app/api/checker/cron/_cron.ts
@@ -88,26 +88,6 @@ export const cron = async ({
const [response] = await client.createTask(request);
allResult.push(response);
-
- /**
- * Pushing to our Golang endpoint
- */
- const tempTask: google.cloud.tasks.v2beta3.ITask = {
- httpRequest: {
- headers: {
- "Content-Type": "application/json", // Set content type to ensure compatibility your application's request parsing
- ...(region !== "auto" && { "fly-prefer-region": region }), // Specify the region you want the request to be sent to
- Authorization: `Basic ${env.CRON_SECRET}`,
- },
- httpMethod: "POST",
- url: "https://checker.openstatus.dev/",
- body: Buffer.from(JSON.stringify(payload)).toString("base64"),
- },
- };
- const tempRequest = { parent: parent, task: tempTask };
- const [tempResponse] = await client.createTask(tempRequest);
-
- allResult.push(tempResponse);
}
}
await Promise.all(allResult);