Skip to content

Open-source status page, incident response and uptime monitoring

fix: cron (#2058)

Thibault Le Ouay67ac7a4bb2c0
diff --git a/apps/workflows/src/cron/checker.ts b/apps/workflows/src/cron/checker.ts
index 3b253d642..da4c62bb8 100644
--- a/apps/workflows/src/cron/checker.ts
+++ b/apps/workflows/src/cron/checker.ts
@@ -1,5 +1,6 @@
import { CloudTasksClient } from "@google-cloud/tasks";
import type { google } from "@google-cloud/tasks/build/protos/protos";
+import { Effect, Either, Schedule } from "effect";
import { z } from "zod";
import { and, eq, gte, isNotNull, lte, notInArray } from "@openstatus/db";
@@ -31,6 +32,15 @@ import {
import type { Context } from "hono";
import { env } from "../env";
+type TaskInput = {
+ row: z.infer<typeof selectMonitorSchema>;
+ timestamp: number;
+ client: CloudTasksClient;
+ parent: string;
+ status: MonitorStatus;
+ region: Region;
+};
+
export const isAuthorizedDomain = (url: string) => {
return url.includes(env().SITE_URL);
};
@@ -107,7 +117,7 @@ export async function sendCheckerTasks(
});
const monitors = z.array(selectMonitorSchema).safeParse(result);
- const allResult = [];
+ const taskInputs: TaskInput[] = [];
if (!monitors.success) {
logger.error(`Error while fetching the monitors ${monitors.error}`);
throw new Error("Error while fetching the monitors");
@@ -146,19 +156,10 @@ export async function sendCheckerTasks(
logger.error(`Deprecated region ${region}`);
continue;
}
- const response = createCronTask({
- row,
- timestamp,
- client,
- parent,
- status,
- region,
- });
- allResult.push(response);
+ taskInputs.push({ row, timestamp, client, parent, status, region });
if (periodicity === "30s") {
- // we schedule another task in 30s
const scheduledAt = timestamp + 30 * 1000;
- const response = createCronTask({
+ taskInputs.push({
row,
timestamp: scheduledAt,
client,
@@ -166,19 +167,53 @@ export async function sendCheckerTasks(
status,
region,
});
- allResult.push(response);
}
}
}
- const allRequests = await Promise.allSettled(allResult);
+ const results = await Effect.runPromise(
+ Effect.forEach(
+ taskInputs,
+ (input) =>
+ Effect.tryPromise({
+ try: () => createCronTask(input),
+ catch: (err) => {
+ if (err instanceof Error && "code" in err && err.code === 6) {
+ return "ALREADY_EXISTS" as const;
+ }
+ return new Error(
+ `Failed creating task for monitor ${input.row.id} in region ${input.region}`,
+ );
+ },
+ }).pipe(
+ Effect.catchIf(
+ (err): err is "ALREADY_EXISTS" => err === "ALREADY_EXISTS",
+ () => Effect.void,
+ ),
+ Effect.retry({
+ times: 3,
+ schedule: Schedule.exponential("1000 millis"),
+ }),
+ Effect.either,
+ ),
+ { concurrency: "unbounded" },
+ ),
+ );
+
+ for (const result of results) {
+ if (Either.isLeft(result)) {
+ logger.error("Task creation failed after retries", {
+ error_message: result.left.message,
+ });
+ }
+ }
- const success = allRequests.filter((r) => r.status === "fulfilled").length;
- const failed = allRequests.filter((r) => r.status === "rejected").length;
+ const success = results.filter(Either.isRight).length;
+ const failed = results.filter(Either.isLeft).length;
logger.info("Completed cron job", {
periodicity,
- total_tasks: allResult.length,
+ total_tasks: taskInputs.length,
success_count: success,
failed_count: failed,
});
@@ -297,7 +332,9 @@ const createCronTask = async ({
if (regionInfo.provider === "railway") {
regionHeader = { "railway-region": region.replace("railway_", "") };
}
+ const taskName = `${parent}/tasks/monitor-${row.id}-${region}-${timestamp}`;
const newTask: google.cloud.tasks.v2beta3.ITask = {
+ name: taskName,
httpRequest: {
headers: {
"Content-Type": "application/json", // Set content type to ensure compatibility your application's request parsing
diff --git a/apps/workflows/src/cron/index.ts b/apps/workflows/src/cron/index.ts
index c3c2071ef..67a6f7cdb 100644
--- a/apps/workflows/src/cron/index.ts
+++ b/apps/workflows/src/cron/index.ts
@@ -1,5 +1,6 @@
import { getSentry } from "@hono/sentry";
import { monitorPeriodicitySchema } from "@openstatus/db/src/schema/constants";
+import { Effect, Schedule } from "effect";
import { Hono } from "hono";
import { env } from "../env";
import { sendCheckerTasks } from "./checker";
@@ -35,24 +36,39 @@ app.get("/checker/:period", async (c) => {
monitorSlug: period,
status: "in_progress",
});
- try {
- await sendCheckerTasks(schema.data, c);
- sentry.captureCheckIn({
- checkInId,
- monitorSlug: period,
- status: "ok",
- });
- return c.json({ success: schema.data }, 200);
- } catch (e) {
- console.error(e);
- sentry.captureMessage(`Error in /checker/${period} cron: ${e}`, "error");
- sentry.captureCheckIn({
- checkInId,
- monitorSlug: period,
- status: "error",
- });
- return c.text("Internal Server Error", 500);
- }
+
+ void Effect.runPromise(
+ Effect.tryPromise({
+ try: () => sendCheckerTasks(schema.data, c),
+ catch: (e) => new Error(`Error in /checker/${period} cron: ${e}`),
+ }).pipe(
+ Effect.retry({
+ times: 3,
+ schedule: Schedule.exponential("1000 millis"),
+ }),
+ Effect.tap(() =>
+ Effect.sync(() =>
+ sentry.captureCheckIn({
+ checkInId,
+ monitorSlug: period,
+ status: "ok",
+ }),
+ ),
+ ),
+ Effect.catchAll((e) =>
+ Effect.sync(() => {
+ console.error(e);
+ sentry.captureMessage(e.message, "error");
+ sentry.captureCheckIn({
+ checkInId,
+ monitorSlug: period,
+ status: "error",
+ });
+ }),
+ ),
+ ),
+ );
+ return c.json({ success: schema.data }, 200);
});
app.get("/emails/follow-up", async (c) => {
@@ -97,23 +113,18 @@ app.get("/monitors/:step", async (c) => {
switch (schema.data) {
case "14days":
- // We send the first email
await Step14Days(Number(userId), Number(initialRun));
break;
case "3days":
await Step3Days(Number(userId), Number(initialRun));
- // 3 days before we send the second email
break;
case "paused":
- // Let's pause the monitor
await StepPaused(Number(userId), Number(initialRun));
break;
default:
throw new Error("Invalid step");
}
- // Swith on step
- // and do the right action
- //
+
return c.json({ success: true }, 200);
});