Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
fix: cron (#2058)
Thibault Le Ouay67ac7a4bb2c0
diff --git a/apps/workflows/src/cron/checker.ts b/apps/workflows/src/cron/checker.tsindex 3b253d642..da4c62bb8 100644--- a/apps/workflows/src/cron/checker.ts+++ b/apps/workflows/src/cron/checker.ts@@ -1,5 +1,6 @@import { CloudTasksClient } from "@google-cloud/tasks";import type { google } from "@google-cloud/tasks/build/protos/protos";+import { Effect, Either, Schedule } from "effect";import { z } from "zod";import { and, eq, gte, isNotNull, lte, notInArray } from "@openstatus/db";@@ -31,6 +32,15 @@ import {import type { Context } from "hono";import { env } from "../env";+type TaskInput = {+ row: z.infer<typeof selectMonitorSchema>;+ timestamp: number;+ client: CloudTasksClient;+ parent: string;+ status: MonitorStatus;+ region: Region;+};+export const isAuthorizedDomain = (url: string) => {return url.includes(env().SITE_URL);};@@ -107,7 +117,7 @@ export async function sendCheckerTasks(});const monitors = z.array(selectMonitorSchema).safeParse(result);- const allResult = [];+ const taskInputs: TaskInput[] = [];if (!monitors.success) {logger.error(`Error while fetching the monitors ${monitors.error}`);throw new Error("Error while fetching the monitors");@@ -146,19 +156,10 @@ export async function sendCheckerTasks(logger.error(`Deprecated region ${region}`);continue;}- const response = createCronTask({- row,- timestamp,- client,- parent,- status,- region,- });- allResult.push(response);+ taskInputs.push({ row, timestamp, client, parent, status, region });if (periodicity === "30s") {- // we schedule another task in 30sconst scheduledAt = timestamp + 30 * 1000;- const response = createCronTask({+ taskInputs.push({row,timestamp: scheduledAt,client,@@ -166,19 +167,53 @@ export async function sendCheckerTasks(status,region,});- allResult.push(response);}}}- const allRequests = await Promise.allSettled(allResult);+ const results = await Effect.runPromise(+ Effect.forEach(+ taskInputs,+ (input) =>+ Effect.tryPromise({+ try: () => createCronTask(input),+ catch: (err) => {+ if (err instanceof Error && "code" in err && err.code === 6) {+ return "ALREADY_EXISTS" as const;+ }+ return new Error(+ `Failed creating task for monitor ${input.row.id} in region ${input.region}`,+ );+ },+ }).pipe(+ Effect.catchIf(+ (err): err is "ALREADY_EXISTS" => err === "ALREADY_EXISTS",+ () => Effect.void,+ ),+ Effect.retry({+ times: 3,+ schedule: Schedule.exponential("1000 millis"),+ }),+ Effect.either,+ ),+ { concurrency: "unbounded" },+ ),+ );++ for (const result of results) {+ if (Either.isLeft(result)) {+ logger.error("Task creation failed after retries", {+ error_message: result.left.message,+ });+ }+ }- const success = allRequests.filter((r) => r.status === "fulfilled").length;- const failed = allRequests.filter((r) => r.status === "rejected").length;+ const success = results.filter(Either.isRight).length;+ const failed = results.filter(Either.isLeft).length;logger.info("Completed cron job", {periodicity,- total_tasks: allResult.length,+ total_tasks: taskInputs.length,success_count: success,failed_count: failed,});@@ -297,7 +332,9 @@ const createCronTask = async ({if (regionInfo.provider === "railway") {regionHeader = { "railway-region": region.replace("railway_", "") };}+ const taskName = `${parent}/tasks/monitor-${row.id}-${region}-${timestamp}`;const newTask: google.cloud.tasks.v2beta3.ITask = {+ name: taskName,httpRequest: {headers: {"Content-Type": "application/json", // Set content type to ensure compatibility your application's request parsingdiff --git a/apps/workflows/src/cron/index.ts b/apps/workflows/src/cron/index.tsindex c3c2071ef..67a6f7cdb 100644--- a/apps/workflows/src/cron/index.ts+++ b/apps/workflows/src/cron/index.ts@@ -1,5 +1,6 @@import { getSentry } from "@hono/sentry";import { monitorPeriodicitySchema } from "@openstatus/db/src/schema/constants";+import { Effect, Schedule } from "effect";import { Hono } from "hono";import { env } from "../env";import { sendCheckerTasks } from "./checker";@@ -35,24 +36,39 @@ app.get("/checker/:period", async (c) => {monitorSlug: period,status: "in_progress",});- try {- await sendCheckerTasks(schema.data, c);- sentry.captureCheckIn({- checkInId,- monitorSlug: period,- status: "ok",- });- return c.json({ success: schema.data }, 200);- } catch (e) {- console.error(e);- sentry.captureMessage(`Error in /checker/${period} cron: ${e}`, "error");- sentry.captureCheckIn({- checkInId,- monitorSlug: period,- status: "error",- });- return c.text("Internal Server Error", 500);- }++ void Effect.runPromise(+ Effect.tryPromise({+ try: () => sendCheckerTasks(schema.data, c),+ catch: (e) => new Error(`Error in /checker/${period} cron: ${e}`),+ }).pipe(+ Effect.retry({+ times: 3,+ schedule: Schedule.exponential("1000 millis"),+ }),+ Effect.tap(() =>+ Effect.sync(() =>+ sentry.captureCheckIn({+ checkInId,+ monitorSlug: period,+ status: "ok",+ }),+ ),+ ),+ Effect.catchAll((e) =>+ Effect.sync(() => {+ console.error(e);+ sentry.captureMessage(e.message, "error");+ sentry.captureCheckIn({+ checkInId,+ monitorSlug: period,+ status: "error",+ });+ }),+ ),+ ),+ );+ return c.json({ success: schema.data }, 200);});app.get("/emails/follow-up", async (c) => {@@ -97,23 +113,18 @@ app.get("/monitors/:step", async (c) => {switch (schema.data) {case "14days":- // We send the first emailawait Step14Days(Number(userId), Number(initialRun));break;case "3days":await Step3Days(Number(userId), Number(initialRun));- // 3 days before we send the second emailbreak;case "paused":- // Let's pause the monitorawait StepPaused(Number(userId), Number(initialRun));break;default:throw new Error("Invalid step");}- // Swith on step- // and do the right action- //+return c.json({ success: true }, 200);});