Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
perf: fix CPU bottlenecks in workflows checker cron (#2061)
Thibault Le Ouayc86a7bde0b65
diff --git a/apps/workflows/.dockerignore b/apps/workflows/.dockerignoreindex d5d76ff57..5e73b32b9 100644--- a/apps/workflows/.dockerignore+++ b/apps/workflows/.dockerignore@@ -1,4 +1,4 @@-# This file is generated by Dofigen v2.7.0+# This file is generated by Dofigen v2.6.0# See https://github.com/lenra-io/dofigennode_modulesdiff --git a/apps/workflows/Dockerfile b/apps/workflows/Dockerfileindex d345aca39..506c737b9 100644--- a/apps/workflows/Dockerfile+++ b/apps/workflows/Dockerfile@@ -1,12 +1,12 @@# syntax=docker/dockerfile:1.19.0-# This file is generated by Dofigen v2.7.0+# This file is generated by Dofigen v2.6.0# See https://github.com/lenra-io/dofigen# docker-FROM oven/bun@sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a AS docker+FROM oven/bun@sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7 AS dockerLABEL \- org.opencontainers.image.base.digest="sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a" \- org.opencontainers.image.base.name="docker.io/oven/bun:1.3.6"+ org.opencontainers.image.base.digest="sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7" \+ org.opencontainers.image.base.name="docker.io/oven/bun:1.3.11"WORKDIR /app/apps/workflowsCOPY \--link \@@ -14,10 +14,10 @@ COPY \RUN bun run src/build-docker.ts# install-FROM oven/bun@sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a AS install+FROM oven/bun@sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7 AS installLABEL \- org.opencontainers.image.base.digest="sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a" \- org.opencontainers.image.base.name="docker.io/oven/bun:1.3.6" \+ org.opencontainers.image.base.digest="sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7" \+ org.opencontainers.image.base.name="docker.io/oven/bun:1.3.11" \org.opencontainers.image.stage="install"WORKDIR /app/RUN \@@ -51,10 +51,10 @@ RUN \bun install --production --frozen-lockfile --verbose# build-FROM oven/bun@sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a AS build+FROM oven/bun@sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7 AS buildLABEL \- org.opencontainers.image.base.digest="sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a" \- org.opencontainers.image.base.name="docker.io/oven/bun:1.3.6" \+ org.opencontainers.image.base.digest="sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7" \+ org.opencontainers.image.base.name="docker.io/oven/bun:1.3.11" \org.opencontainers.image.stage="build"ENV NODE_ENV="production"WORKDIR /app/apps/workflows@@ -68,10 +68,10 @@ COPY \RUN bun build --compile --target bun --sourcemap src/index.ts --outfile=app# libsql-FROM oven/bun@sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a AS libsql+FROM oven/bun@sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7 AS libsqlLABEL \- org.opencontainers.image.base.digest="sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a" \- org.opencontainers.image.base.name="docker.io/oven/bun:1.3.6"+ org.opencontainers.image.base.digest="sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7" \+ org.opencontainers.image.base.name="docker.io/oven/bun:1.3.11"WORKDIR /app/COPY \--from=docker \@@ -82,7 +82,7 @@ RUN bun install# runtimeFROM debian@sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734 AS runtimeLABEL \- io.dofigen.version="2.7.0" \+ io.dofigen.version="2.6.0" \org.opencontainers.image.authors="OpenStatus Team" \org.opencontainers.image.base.digest="sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734" \org.opencontainers.image.base.name="docker.io/debian:bullseye-slim" \diff --git a/apps/workflows/dofigen.lock b/apps/workflows/dofigen.lockindex 389dac8ff..00f777e83 100644--- a/apps/workflows/dofigen.lock+++ b/apps/workflows/dofigen.lock@@ -12,42 +12,13 @@ effective: |- /packages/error- /packages/trackerbuilders:- libsql:- fromImage:- path: oven/bun- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a- label:- org.opencontainers.image.base.digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a- org.opencontainers.image.base.name: docker.io/oven/bun:1.3.6- workdir: /app/- copy:- - fromBuilder: docker- paths:- - /app/apps/build-docker/package.json- target: /app/package.json- run:- - bun install- docker:- fromImage:- path: oven/bun- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a- label:- org.opencontainers.image.base.name: docker.io/oven/bun:1.3.6- org.opencontainers.image.base.digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a- workdir: /app/apps/workflows- copy:- - paths:- - .- target: /app/- run:- - bun run src/build-docker.tsinstall:fromImage:path: oven/bun- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7label:- org.opencontainers.image.base.digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a- org.opencontainers.image.base.name: docker.io/oven/bun:1.3.6+ org.opencontainers.image.base.name: docker.io/oven/bun:1.3.11+ org.opencontainers.image.base.digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7org.opencontainers.image.stage: installworkdir: /app/run:@@ -107,14 +78,43 @@ effective: |source: packages/theme-store/package.json- target: packages/locales/package.jsonsource: packages/locales/package.json+ docker:+ fromImage:+ path: oven/bun+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7+ label:+ org.opencontainers.image.base.name: docker.io/oven/bun:1.3.11+ org.opencontainers.image.base.digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7+ workdir: /app/apps/workflows+ copy:+ - paths:+ - .+ target: /app/+ run:+ - bun run src/build-docker.ts+ libsql:+ fromImage:+ path: oven/bun+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7+ label:+ org.opencontainers.image.base.name: docker.io/oven/bun:1.3.11+ org.opencontainers.image.base.digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7+ workdir: /app/+ copy:+ - fromBuilder: docker+ paths:+ - /app/apps/build-docker/package.json+ target: /app/package.json+ run:+ - bun installbuild:fromImage:path: oven/bun- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7label:org.opencontainers.image.stage: build- org.opencontainers.image.base.name: docker.io/oven/bun:1.3.6- org.opencontainers.image.base.digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a+ org.opencontainers.image.base.name: docker.io/oven/bun:1.3.11+ org.opencontainers.image.base.digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7workdir: /app/apps/workflowsenv:NODE_ENV: production@@ -132,14 +132,14 @@ effective: |path: debiandigest: sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734label:- org.opencontainers.image.vendor: OpenStatus- org.opencontainers.image.base.name: docker.io/debian:bullseye-slim- org.opencontainers.image.source: https://github.com/openstatusHQ/openstatus- io.dofigen.version: 2.7.0- org.opencontainers.image.title: OpenStatus Workflows- org.opencontainers.image.description: Background job processing and probe scheduling for OpenStatusorg.opencontainers.image.base.digest: sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734org.opencontainers.image.authors: OpenStatus Team+ org.opencontainers.image.source: https://github.com/openstatusHQ/openstatus+ org.opencontainers.image.description: Background job processing and probe scheduling for OpenStatus+ org.opencontainers.image.title: OpenStatus Workflows+ org.opencontainers.image.vendor: OpenStatus+ org.opencontainers.image.base.name: docker.io/debian:bullseye-slim+ io.dofigen.version: 2.6.0workdir: /app/copy:- fromBuilder: build@@ -170,11 +170,11 @@ images:digest: sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734oven:bun:- 1.3.6:- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a+ 1.3.11:+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7resources:dofigen.yml:- hash: f3341585d9e20ed6c82f7ba457691a460a7f6a7c4378b5bec8483ee11854d309+ hash: af5f6cc6d868908681abdb8a70ec8c09594d385545750b5c157d192313d9a3e4content: |ignore:- node_modules@@ -190,7 +190,7 @@ resources:- /packages/trackerbuilders:install:- fromImage: oven/bun:1.3.6+ fromImage: oven/bun:1.3.11workdir: /app/labels:org.opencontainers.image.stage: install@@ -228,7 +228,7 @@ resources:# Stage 4: Build application (compile to binary)build:- fromImage: oven/bun:1.3.6+ fromImage: oven/bun:1.3.11workdir: /app/apps/workflowslabels:org.opencontainers.image.stage: build@@ -244,14 +244,14 @@ resources:run: bun build --compile --target bun --sourcemap src/index.ts --outfile=appdocker:- fromImage: oven/bun:1.3.6+ fromImage: oven/bun:1.3.11workdir: /app/apps/workflowscopy:- . /app/run: bun run src/build-docker.tslibsql:- fromImage: oven/bun:1.3.6+ fromImage: oven/bun:1.3.11workdir: /app/copy:- fromBuilder: dockerdiff --git a/apps/workflows/dofigen.yml b/apps/workflows/dofigen.ymlindex e1d3d5538..7f8b6fd6a 100644--- a/apps/workflows/dofigen.yml+++ b/apps/workflows/dofigen.yml@@ -12,7 +12,7 @@ ignore:- /packages/trackerbuilders:install:- fromImage: oven/bun:1.3.6+ fromImage: oven/bun:1.3.11workdir: /app/labels:org.opencontainers.image.stage: install@@ -50,7 +50,7 @@ builders:# Stage 4: Build application (compile to binary)build:- fromImage: oven/bun:1.3.6+ fromImage: oven/bun:1.3.11workdir: /app/apps/workflowslabels:org.opencontainers.image.stage: build@@ -66,14 +66,14 @@ builders:run: bun build --compile --target bun --sourcemap src/index.ts --outfile=appdocker:- fromImage: oven/bun:1.3.6+ fromImage: oven/bun:1.3.11workdir: /app/apps/workflowscopy:- . /app/run: bun run src/build-docker.tslibsql:- fromImage: oven/bun:1.3.6+ fromImage: oven/bun:1.3.11workdir: /app/copy:- fromBuilder: dockerdiff --git a/apps/workflows/src/cron/checker.ts b/apps/workflows/src/cron/checker.tsindex da4c62bb8..6f6483dc0 100644--- a/apps/workflows/src/cron/checker.ts+++ b/apps/workflows/src/cron/checker.ts@@ -3,7 +3,15 @@ import type { google } from "@google-cloud/tasks/build/protos/protos";import { Effect, Either, Schedule } from "effect";import { z } from "zod";-import { and, eq, gte, isNotNull, lte, notInArray } from "@openstatus/db";+import {+ and,+ eq,+ gte,+ inArray,+ isNotNull,+ lte,+ notInArray,+} from "@openstatus/db";import {type MonitorStatus,maintenance,@@ -35,8 +43,6 @@ import { env } from "../env";type TaskInput = {row: z.infer<typeof selectMonitorSchema>;timestamp: number;- client: CloudTasksClient;- parent: string;status: MonitorStatus;region: Region;};@@ -47,29 +53,19 @@ export const isAuthorizedDomain = (url: string) => {const logger = getLogger("workflow");-const channelOptions = {- // Conservative 5-minute keepalive (gRPC best practice)- "grpc.keepalive_time_ms": 300000,- // 5-second timeout sufficient for ping response- "grpc.keepalive_timeout_ms": 5000,- // Disable pings without active calls to avoid server conflicts- "grpc.keepalive_permit_without_calls": 1,-};+const client = new CloudTasksClient({+ fallback: "rest",+ projectId: env().GCP_PROJECT_ID,+ credentials: {+ client_email: env().GCP_CLIENT_EMAIL,+ private_key: env().GCP_PRIVATE_KEY.replaceAll("\\n", "\n"),+ },+});export async function sendCheckerTasks(periodicity: z.infer<typeof monitorPeriodicitySchema>,c: Context,) {- const client = new CloudTasksClient({- fallback: "rest",- channelOptions,- projectId: env().GCP_PROJECT_ID,- credentials: {- client_email: env().GCP_CLIENT_EMAIL,- private_key: env().GCP_PRIVATE_KEY.replaceAll("\\n", "\n"),- },- });-const parent = client.queuePath(env().GCP_PROJECT_ID,env().GCP_LOCATION,@@ -123,26 +119,43 @@ export async function sendCheckerTasks(throw new Error("Error while fetching the monitors");}- for (const row of monitors.data) {- // const selectedRegions = row.regions.length > 0 ? row.regions : ["ams"];+ if (monitors.data.length === 0) {+ logger.info("No monitors to check", { periodicity });+ return;+ }++ // Batch fetch all monitor statuses in a single query (N+1 fix)+ const monitorIds = monitors.data.map((m) => m.id);+ const rawStatuses = await db+ .select()+ .from(monitorStatusTable)+ .where(inArray(monitorStatusTable.monitorId, monitorIds))+ .all();- const result = await db- .select()- .from(monitorStatusTable)- .where(eq(monitorStatusTable.monitorId, row.id))- .all();- const monitorStatus = z.array(selectMonitorStatusSchema).safeParse(result);- if (!monitorStatus.success) {- logger.error("Failed to parse monitor status", {- monitor_id: row.id,- error_message: monitorStatus.error.message,+ const statusMap = new Map<+ number,+ z.infer<typeof selectMonitorStatusSchema>[]+ >();+ for (const raw of rawStatuses) {+ const parsed = selectMonitorStatusSchema.safeParse(raw);+ if (!parsed.success) {+ logger.error("Failed to parse monitor status row", {+ monitor_id: raw.monitorId,+ error_message: parsed.error.message,});continue;}+ const list = statusMap.get(raw.monitorId) ?? [];+ list.push(parsed.data);+ statusMap.set(raw.monitorId, list);+ }++ for (const row of monitors.data) {+ const monitorStatuses = statusMap.get(row.id) ?? [];for (const region of row.regions) {const status =- monitorStatus.data.find((m) => region === m.region)?.status || "active";+ monitorStatuses.find((m) => region === m.region)?.status || "active";const r = regionDict[region as keyof typeof regionDict];@@ -156,14 +169,12 @@ export async function sendCheckerTasks(logger.error(`Deprecated region ${region}`);continue;}- taskInputs.push({ row, timestamp, client, parent, status, region });+ taskInputs.push({ row, timestamp, status, region });if (periodicity === "30s") {const scheduledAt = timestamp + 30 * 1000;taskInputs.push({row,timestamp: scheduledAt,- client,- parent,status,region,});@@ -176,7 +187,7 @@ export async function sendCheckerTasks(taskInputs,(input) =>Effect.tryPromise({- try: () => createCronTask(input),+ try: () => createCronTask(input, parent),catch: (err) => {if (err instanceof Error && "code" in err && err.code === 6) {return "ALREADY_EXISTS" as const;@@ -196,7 +207,7 @@ export async function sendCheckerTasks(}),Effect.either,),- { concurrency: "unbounded" },+ { concurrency: 100 },),);@@ -216,6 +227,7 @@ export async function sendCheckerTasks(total_tasks: taskInputs.length,success_count: success,failed_count: failed,+ duration_ms: Date.now() - timestamp,});if (failed > 0) {logger.error("Cron job had failures", {@@ -230,21 +242,10 @@ export async function sendCheckerTasks(}}// timestamp needs to be in ms-const createCronTask = async ({- row,- timestamp,- client,- parent,- status,- region,-}: {- row: z.infer<typeof selectMonitorSchema>;- timestamp: number;- client: CloudTasksClient;- parent: string;- status: MonitorStatus;- region: Region;-}) => {+const createCronTask = async (+ { row, timestamp, status, region }: TaskInput,+ parent: string,+) => {let payload:| z.infer<typeof httpPayloadSchema>| z.infer<typeof tpcPayloadSchema>