Skip to content

Open-source status page, incident response and uptime monitoring

perf: fix CPU bottlenecks in workflows checker cron (#2061)

Thibault Le Ouayc86a7bde0b65
diff --git a/apps/workflows/.dockerignore b/apps/workflows/.dockerignore
index d5d76ff57..5e73b32b9 100644
--- a/apps/workflows/.dockerignore
+++ b/apps/workflows/.dockerignore
@@ -1,4 +1,4 @@
-# This file is generated by Dofigen v2.7.0
+# This file is generated by Dofigen v2.6.0
# See https://github.com/lenra-io/dofigen
node_modules
diff --git a/apps/workflows/Dockerfile b/apps/workflows/Dockerfile
index d345aca39..506c737b9 100644
--- a/apps/workflows/Dockerfile
+++ b/apps/workflows/Dockerfile
@@ -1,12 +1,12 @@
# syntax=docker/dockerfile:1.19.0
-# This file is generated by Dofigen v2.7.0
+# This file is generated by Dofigen v2.6.0
# See https://github.com/lenra-io/dofigen
# docker
-FROM oven/bun@sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a AS docker
+FROM oven/bun@sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7 AS docker
LABEL \
- org.opencontainers.image.base.digest="sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a" \
- org.opencontainers.image.base.name="docker.io/oven/bun:1.3.6"
+ org.opencontainers.image.base.digest="sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7" \
+ org.opencontainers.image.base.name="docker.io/oven/bun:1.3.11"
WORKDIR /app/apps/workflows
COPY \
--link \
@@ -14,10 +14,10 @@ COPY \
RUN bun run src/build-docker.ts
# install
-FROM oven/bun@sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a AS install
+FROM oven/bun@sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7 AS install
LABEL \
- org.opencontainers.image.base.digest="sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a" \
- org.opencontainers.image.base.name="docker.io/oven/bun:1.3.6" \
+ org.opencontainers.image.base.digest="sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7" \
+ org.opencontainers.image.base.name="docker.io/oven/bun:1.3.11" \
org.opencontainers.image.stage="install"
WORKDIR /app/
RUN \
@@ -51,10 +51,10 @@ RUN \
bun install --production --frozen-lockfile --verbose
# build
-FROM oven/bun@sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a AS build
+FROM oven/bun@sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7 AS build
LABEL \
- org.opencontainers.image.base.digest="sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a" \
- org.opencontainers.image.base.name="docker.io/oven/bun:1.3.6" \
+ org.opencontainers.image.base.digest="sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7" \
+ org.opencontainers.image.base.name="docker.io/oven/bun:1.3.11" \
org.opencontainers.image.stage="build"
ENV NODE_ENV="production"
WORKDIR /app/apps/workflows
@@ -68,10 +68,10 @@ COPY \
RUN bun build --compile --target bun --sourcemap src/index.ts --outfile=app
# libsql
-FROM oven/bun@sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a AS libsql
+FROM oven/bun@sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7 AS libsql
LABEL \
- org.opencontainers.image.base.digest="sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a" \
- org.opencontainers.image.base.name="docker.io/oven/bun:1.3.6"
+ org.opencontainers.image.base.digest="sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7" \
+ org.opencontainers.image.base.name="docker.io/oven/bun:1.3.11"
WORKDIR /app/
COPY \
--from=docker \
@@ -82,7 +82,7 @@ RUN bun install
# runtime
FROM debian@sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734 AS runtime
LABEL \
- io.dofigen.version="2.7.0" \
+ io.dofigen.version="2.6.0" \
org.opencontainers.image.authors="OpenStatus Team" \
org.opencontainers.image.base.digest="sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734" \
org.opencontainers.image.base.name="docker.io/debian:bullseye-slim" \
diff --git a/apps/workflows/dofigen.lock b/apps/workflows/dofigen.lock
index 389dac8ff..00f777e83 100644
--- a/apps/workflows/dofigen.lock
+++ b/apps/workflows/dofigen.lock
@@ -12,42 +12,13 @@ effective: |
- /packages/error
- /packages/tracker
builders:
- libsql:
- fromImage:
- path: oven/bun
- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a
- label:
- org.opencontainers.image.base.digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a
- org.opencontainers.image.base.name: docker.io/oven/bun:1.3.6
- workdir: /app/
- copy:
- - fromBuilder: docker
- paths:
- - /app/apps/build-docker/package.json
- target: /app/package.json
- run:
- - bun install
- docker:
- fromImage:
- path: oven/bun
- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a
- label:
- org.opencontainers.image.base.name: docker.io/oven/bun:1.3.6
- org.opencontainers.image.base.digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a
- workdir: /app/apps/workflows
- copy:
- - paths:
- - .
- target: /app/
- run:
- - bun run src/build-docker.ts
install:
fromImage:
path: oven/bun
- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a
+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7
label:
- org.opencontainers.image.base.digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a
- org.opencontainers.image.base.name: docker.io/oven/bun:1.3.6
+ org.opencontainers.image.base.name: docker.io/oven/bun:1.3.11
+ org.opencontainers.image.base.digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7
org.opencontainers.image.stage: install
workdir: /app/
run:
@@ -107,14 +78,43 @@ effective: |
source: packages/theme-store/package.json
- target: packages/locales/package.json
source: packages/locales/package.json
+ docker:
+ fromImage:
+ path: oven/bun
+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7
+ label:
+ org.opencontainers.image.base.name: docker.io/oven/bun:1.3.11
+ org.opencontainers.image.base.digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7
+ workdir: /app/apps/workflows
+ copy:
+ - paths:
+ - .
+ target: /app/
+ run:
+ - bun run src/build-docker.ts
+ libsql:
+ fromImage:
+ path: oven/bun
+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7
+ label:
+ org.opencontainers.image.base.name: docker.io/oven/bun:1.3.11
+ org.opencontainers.image.base.digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7
+ workdir: /app/
+ copy:
+ - fromBuilder: docker
+ paths:
+ - /app/apps/build-docker/package.json
+ target: /app/package.json
+ run:
+ - bun install
build:
fromImage:
path: oven/bun
- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a
+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7
label:
org.opencontainers.image.stage: build
- org.opencontainers.image.base.name: docker.io/oven/bun:1.3.6
- org.opencontainers.image.base.digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a
+ org.opencontainers.image.base.name: docker.io/oven/bun:1.3.11
+ org.opencontainers.image.base.digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7
workdir: /app/apps/workflows
env:
NODE_ENV: production
@@ -132,14 +132,14 @@ effective: |
path: debian
digest: sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734
label:
- org.opencontainers.image.vendor: OpenStatus
- org.opencontainers.image.base.name: docker.io/debian:bullseye-slim
- org.opencontainers.image.source: https://github.com/openstatusHQ/openstatus
- io.dofigen.version: 2.7.0
- org.opencontainers.image.title: OpenStatus Workflows
- org.opencontainers.image.description: Background job processing and probe scheduling for OpenStatus
org.opencontainers.image.base.digest: sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734
org.opencontainers.image.authors: OpenStatus Team
+ org.opencontainers.image.source: https://github.com/openstatusHQ/openstatus
+ org.opencontainers.image.description: Background job processing and probe scheduling for OpenStatus
+ org.opencontainers.image.title: OpenStatus Workflows
+ org.opencontainers.image.vendor: OpenStatus
+ org.opencontainers.image.base.name: docker.io/debian:bullseye-slim
+ io.dofigen.version: 2.6.0
workdir: /app/
copy:
- fromBuilder: build
@@ -170,11 +170,11 @@ images:
digest: sha256:b32674fb57780ad57d7b0749242d3f585f462f4ec4a60ae0adacd945f9cb9734
oven:
bun:
- 1.3.6:
- digest: sha256:f20d9cf365ab35529384f1717687c739c92e6f39157a35a95ef06f4049a10e4a
+ 1.3.11:
+ digest: sha256:0733e50325078969732ebe3b15ce4c4be5082f18c4ac1a0f0ca4839c2e4e42a7
resources:
dofigen.yml:
- hash: f3341585d9e20ed6c82f7ba457691a460a7f6a7c4378b5bec8483ee11854d309
+ hash: af5f6cc6d868908681abdb8a70ec8c09594d385545750b5c157d192313d9a3e4
content: |
ignore:
- node_modules
@@ -190,7 +190,7 @@ resources:
- /packages/tracker
builders:
install:
- fromImage: oven/bun:1.3.6
+ fromImage: oven/bun:1.3.11
workdir: /app/
labels:
org.opencontainers.image.stage: install
@@ -228,7 +228,7 @@ resources:
# Stage 4: Build application (compile to binary)
build:
- fromImage: oven/bun:1.3.6
+ fromImage: oven/bun:1.3.11
workdir: /app/apps/workflows
labels:
org.opencontainers.image.stage: build
@@ -244,14 +244,14 @@ resources:
run: bun build --compile --target bun --sourcemap src/index.ts --outfile=app
docker:
- fromImage: oven/bun:1.3.6
+ fromImage: oven/bun:1.3.11
workdir: /app/apps/workflows
copy:
- . /app/
run: bun run src/build-docker.ts
libsql:
- fromImage: oven/bun:1.3.6
+ fromImage: oven/bun:1.3.11
workdir: /app/
copy:
- fromBuilder: docker
diff --git a/apps/workflows/dofigen.yml b/apps/workflows/dofigen.yml
index e1d3d5538..7f8b6fd6a 100644
--- a/apps/workflows/dofigen.yml
+++ b/apps/workflows/dofigen.yml
@@ -12,7 +12,7 @@ ignore:
- /packages/tracker
builders:
install:
- fromImage: oven/bun:1.3.6
+ fromImage: oven/bun:1.3.11
workdir: /app/
labels:
org.opencontainers.image.stage: install
@@ -50,7 +50,7 @@ builders:
# Stage 4: Build application (compile to binary)
build:
- fromImage: oven/bun:1.3.6
+ fromImage: oven/bun:1.3.11
workdir: /app/apps/workflows
labels:
org.opencontainers.image.stage: build
@@ -66,14 +66,14 @@ builders:
run: bun build --compile --target bun --sourcemap src/index.ts --outfile=app
docker:
- fromImage: oven/bun:1.3.6
+ fromImage: oven/bun:1.3.11
workdir: /app/apps/workflows
copy:
- . /app/
run: bun run src/build-docker.ts
libsql:
- fromImage: oven/bun:1.3.6
+ fromImage: oven/bun:1.3.11
workdir: /app/
copy:
- fromBuilder: docker
diff --git a/apps/workflows/src/cron/checker.ts b/apps/workflows/src/cron/checker.ts
index da4c62bb8..6f6483dc0 100644
--- a/apps/workflows/src/cron/checker.ts
+++ b/apps/workflows/src/cron/checker.ts
@@ -3,7 +3,15 @@ import type { google } from "@google-cloud/tasks/build/protos/protos";
import { Effect, Either, Schedule } from "effect";
import { z } from "zod";
-import { and, eq, gte, isNotNull, lte, notInArray } from "@openstatus/db";
+import {
+ and,
+ eq,
+ gte,
+ inArray,
+ isNotNull,
+ lte,
+ notInArray,
+} from "@openstatus/db";
import {
type MonitorStatus,
maintenance,
@@ -35,8 +43,6 @@ import { env } from "../env";
type TaskInput = {
row: z.infer<typeof selectMonitorSchema>;
timestamp: number;
- client: CloudTasksClient;
- parent: string;
status: MonitorStatus;
region: Region;
};
@@ -47,29 +53,19 @@ export const isAuthorizedDomain = (url: string) => {
const logger = getLogger("workflow");
-const channelOptions = {
- // Conservative 5-minute keepalive (gRPC best practice)
- "grpc.keepalive_time_ms": 300000,
- // 5-second timeout sufficient for ping response
- "grpc.keepalive_timeout_ms": 5000,
- // Disable pings without active calls to avoid server conflicts
- "grpc.keepalive_permit_without_calls": 1,
-};
+const client = new CloudTasksClient({
+ fallback: "rest",
+ projectId: env().GCP_PROJECT_ID,
+ credentials: {
+ client_email: env().GCP_CLIENT_EMAIL,
+ private_key: env().GCP_PRIVATE_KEY.replaceAll("\\n", "\n"),
+ },
+});
export async function sendCheckerTasks(
periodicity: z.infer<typeof monitorPeriodicitySchema>,
c: Context,
) {
- const client = new CloudTasksClient({
- fallback: "rest",
- channelOptions,
- projectId: env().GCP_PROJECT_ID,
- credentials: {
- client_email: env().GCP_CLIENT_EMAIL,
- private_key: env().GCP_PRIVATE_KEY.replaceAll("\\n", "\n"),
- },
- });
-
const parent = client.queuePath(
env().GCP_PROJECT_ID,
env().GCP_LOCATION,
@@ -123,26 +119,43 @@ export async function sendCheckerTasks(
throw new Error("Error while fetching the monitors");
}
- for (const row of monitors.data) {
- // const selectedRegions = row.regions.length > 0 ? row.regions : ["ams"];
+ if (monitors.data.length === 0) {
+ logger.info("No monitors to check", { periodicity });
+ return;
+ }
+
+ // Batch fetch all monitor statuses in a single query (N+1 fix)
+ const monitorIds = monitors.data.map((m) => m.id);
+ const rawStatuses = await db
+ .select()
+ .from(monitorStatusTable)
+ .where(inArray(monitorStatusTable.monitorId, monitorIds))
+ .all();
- const result = await db
- .select()
- .from(monitorStatusTable)
- .where(eq(monitorStatusTable.monitorId, row.id))
- .all();
- const monitorStatus = z.array(selectMonitorStatusSchema).safeParse(result);
- if (!monitorStatus.success) {
- logger.error("Failed to parse monitor status", {
- monitor_id: row.id,
- error_message: monitorStatus.error.message,
+ const statusMap = new Map<
+ number,
+ z.infer<typeof selectMonitorStatusSchema>[]
+ >();
+ for (const raw of rawStatuses) {
+ const parsed = selectMonitorStatusSchema.safeParse(raw);
+ if (!parsed.success) {
+ logger.error("Failed to parse monitor status row", {
+ monitor_id: raw.monitorId,
+ error_message: parsed.error.message,
});
continue;
}
+ const list = statusMap.get(raw.monitorId) ?? [];
+ list.push(parsed.data);
+ statusMap.set(raw.monitorId, list);
+ }
+
+ for (const row of monitors.data) {
+ const monitorStatuses = statusMap.get(row.id) ?? [];
for (const region of row.regions) {
const status =
- monitorStatus.data.find((m) => region === m.region)?.status || "active";
+ monitorStatuses.find((m) => region === m.region)?.status || "active";
const r = regionDict[region as keyof typeof regionDict];
@@ -156,14 +169,12 @@ export async function sendCheckerTasks(
logger.error(`Deprecated region ${region}`);
continue;
}
- taskInputs.push({ row, timestamp, client, parent, status, region });
+ taskInputs.push({ row, timestamp, status, region });
if (periodicity === "30s") {
const scheduledAt = timestamp + 30 * 1000;
taskInputs.push({
row,
timestamp: scheduledAt,
- client,
- parent,
status,
region,
});
@@ -176,7 +187,7 @@ export async function sendCheckerTasks(
taskInputs,
(input) =>
Effect.tryPromise({
- try: () => createCronTask(input),
+ try: () => createCronTask(input, parent),
catch: (err) => {
if (err instanceof Error && "code" in err && err.code === 6) {
return "ALREADY_EXISTS" as const;
@@ -196,7 +207,7 @@ export async function sendCheckerTasks(
}),
Effect.either,
),
- { concurrency: "unbounded" },
+ { concurrency: 100 },
),
);
@@ -216,6 +227,7 @@ export async function sendCheckerTasks(
total_tasks: taskInputs.length,
success_count: success,
failed_count: failed,
+ duration_ms: Date.now() - timestamp,
});
if (failed > 0) {
logger.error("Cron job had failures", {
@@ -230,21 +242,10 @@ export async function sendCheckerTasks(
}
}
// timestamp needs to be in ms
-const createCronTask = async ({
- row,
- timestamp,
- client,
- parent,
- status,
- region,
-}: {
- row: z.infer<typeof selectMonitorSchema>;
- timestamp: number;
- client: CloudTasksClient;
- parent: string;
- status: MonitorStatus;
- region: Region;
-}) => {
+const createCronTask = async (
+ { row, timestamp, status, region }: TaskInput,
+ parent: string,
+) => {
let payload:
| z.infer<typeof httpPayloadSchema>
| z.infer<typeof tpcPayloadSchema>