Skip to content

Open-source status page, incident response and uptime monitoring

fix: public status api cache (#554)

Maximilian Kaske63ab0a78c1cd
diff --git a/apps/server/src/public/status.ts b/apps/server/src/public/status.ts
index aeb6b62e4..5267ccac8 100644
--- a/apps/server/src/public/status.ts
+++ b/apps/server/src/public/status.ts
@@ -1,7 +1,7 @@
import { Hono } from "hono";
import { endTime, setMetric, startTime } from "hono/timing";
-import { and, db, eq } from "@openstatus/db";
+import { and, db, eq, inArray } from "@openstatus/db";
import {
monitor,
monitorsToPages,
@@ -10,10 +10,11 @@ import {
pagesToStatusReports,
statusReport,
} from "@openstatus/db/src/schema";
-import { getMonitorList, Tinybird } from "@openstatus/tinybird";
+import { getPublicStatus, Tinybird } from "@openstatus/tinybird";
import { Redis } from "@openstatus/upstash";
import { env } from "../env";
+import { notEmpty } from "../utils/not-empty";
// TODO: include ratelimiting
@@ -43,46 +44,22 @@ status.get("/:slug", async (c) => {
}
startTime(c, "database");
-
- const monitorData = await db
- .select()
- .from(monitorsToPages)
- .leftJoin(monitor, and(eq(monitorsToPages.monitorId, monitor.id)))
- .leftJoin(
- monitorsToStatusReport,
- eq(monitor.id, monitorsToStatusReport.monitorId),
- )
- .leftJoin(
- statusReport,
- eq(monitorsToStatusReport.statusReportId, statusReport.id),
- )
- .leftJoin(page, eq(monitorsToPages.pageId, page.id))
- .where(eq(page.slug, slug)) // TODO: query only active monitors
- .all();
-
- const pageStatusReportData = await db
- .select()
- .from(pagesToStatusReports)
- .leftJoin(
- statusReport,
- eq(pagesToStatusReports.statusReportId, statusReport.id),
- )
- .leftJoin(page, eq(pagesToStatusReports.pageId, page.id))
- .where(eq(page.slug, slug))
- .all();
-
- const isIncident = [...pageStatusReportData, ...monitorData].some((data) => {
- if (!data.status_report) return false;
- return !["monitoring", "resolved"].includes(data.status_report.status);
- });
+ const { monitorData, pageStatusReportData, monitorStatusReportData } =
+ await getStatusPageData(slug);
+ endTime(c, "database");
+
+ const isIncident = [...pageStatusReportData, ...monitorStatusReportData].some(
+ (data) => {
+ if (!data.status_report) return false;
+ return !["monitoring", "resolved"].includes(data.status_report.status);
+ },
+ );
startTime(c, "clickhouse");
- // { data: [{ ok, count }] }
const lastMonitorPings = await Promise.allSettled(
monitorData.map(async ({ monitors_to_pages }) => {
- return await getMonitorList(tb)({
+ return await getPublicStatus(tb)({
monitorId: String(monitors_to_pages.monitorId),
- limit: 5, // limits the grouped cronTimestamps
});
}),
);
@@ -111,7 +88,7 @@ status.get("/:slug", async (c) => {
const status: Status = isIncident ? Status.Incident : getStatus(ratio);
- await redis.set(slug, status, { ex: 30 });
+ await redis.set(slug, status, { ex: 60 }); // 1m cache
return c.json({ status });
});
@@ -124,3 +101,47 @@ function getStatus(ratio: number) {
if (ratio >= 0) return Status.MajorOutage;
return Status.Unknown;
}
+
+async function getStatusPageData(slug: string) {
+ const monitorData = await db
+ .select()
+ .from(monitorsToPages)
+ .leftJoin(
+ monitor,
+ // REMINDER: query only active monitors as they are the ones that are displayed on the status page
+ and(eq(monitorsToPages.monitorId, monitor.id), eq(monitor.active, true)),
+ )
+ .leftJoin(page, eq(monitorsToPages.pageId, page.id))
+ .where(eq(page.slug, slug))
+ .all();
+
+ const monitorIds = monitorData.map((i) => i.monitor?.id).filter(notEmpty);
+
+ const monitorStatusReportData = await db
+ .select()
+ .from(monitorsToStatusReport)
+ .leftJoin(
+ statusReport,
+ eq(monitorsToStatusReport.statusReportId, statusReport.id),
+ )
+ .where(inArray(monitorsToStatusReport.monitorId, monitorIds))
+ .all();
+
+ // REMINDER: the query can overlap with the previous one
+ const pageStatusReportData = await db
+ .select()
+ .from(pagesToStatusReports)
+ .leftJoin(
+ statusReport,
+ eq(pagesToStatusReports.statusReportId, statusReport.id),
+ )
+ .leftJoin(page, eq(pagesToStatusReports.pageId, page.id))
+ .where(eq(page.slug, slug))
+ .all();
+
+ return {
+ monitorData,
+ pageStatusReportData,
+ monitorStatusReportData,
+ };
+}
diff --git a/apps/server/src/utils/not-empty.ts b/apps/server/src/utils/not-empty.ts
new file mode 100644
index 000000000..0670766c8
--- /dev/null
+++ b/apps/server/src/utils/not-empty.ts
@@ -0,0 +1,5 @@
+export function notEmpty<TValue>(
+ value: TValue | null | undefined,
+): value is TValue {
+ return value !== null && value !== undefined;
+}
diff --git a/packages/tinybird/pipes/public_status.pipe b/packages/tinybird/pipes/public_status.pipe
new file mode 100644
index 000000000..5554c2ca0
--- /dev/null
+++ b/packages/tinybird/pipes/public_status.pipe
@@ -0,0 +1,23 @@
+VERSION 0
+
+DESCRIPTION >
+ last 5 cron timestamps within last 3 hours
+
+NODE group_by_cronTimestamp
+SQL >
+
+ %
+ SELECT
+ cronTimestamp,
+ count() AS count,
+ count(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok
+ FROM ping_response_v5
+ WHERE
+ monitorId = {{ String(monitorId, '1') }}
+ AND cronTimestamp
+ >= toUnixTimestamp64Milli(toDateTime64(now() - INTERVAL 3 HOUR, 3))
+ GROUP BY cronTimestamp, monitorId
+ ORDER BY cronTimestamp DESC
+ LIMIT {{ Int16(limit, 5)}}
+
+
diff --git a/packages/tinybird/src/client.ts b/packages/tinybird/src/client.ts
index 4e9675881..b592674c2 100644
--- a/packages/tinybird/src/client.ts
+++ b/packages/tinybird/src/client.ts
@@ -3,10 +3,12 @@ import { Tinybird } from "@chronark/zod-bird";
import {
tbBuildHomeStats,
tbBuildMonitorList,
+ tbBuildPublicStatus,
tbBuildResponseList,
tbIngestPingResponse,
tbParameterHomeStats,
tbParameterMonitorList,
+ tbParameterPublicStatus,
tbParameterResponseList,
} from "./validation";
@@ -71,3 +73,11 @@ export function getHomeStats(tb: Tinybird) {
},
});
}
+
+export function getPublicStatus(tb: Tinybird) {
+ return tb.buildPipe({
+ pipe: "public_status__v0",
+ parameters: tbParameterPublicStatus,
+ data: tbBuildPublicStatus,
+ });
+}
diff --git a/packages/tinybird/src/validation.ts b/packages/tinybird/src/validation.ts
index 23d9cf79f..ef45a19b3 100644
--- a/packages/tinybird/src/validation.ts
+++ b/packages/tinybird/src/validation.ts
@@ -81,6 +81,23 @@ export const tbBuildHomeStats = z.object({
count: z.number().int(),
});
+/**
+ * Params for pipe public_status (used for our API /public/status/[slug])
+ */
+export const tbParameterPublicStatus = z.object({
+ monitorId: z.string(),
+ limit: z.number().int().default(5).optional(), // 5 last cronTimestamps
+});
+
+/**
+ * Values from the pipe public_status (used for our API /public/status/[slug])
+ */
+export const tbBuildPublicStatus = z.object({
+ ok: z.number().int(),
+ count: z.number().int(),
+ cronTimestamp: z.number().int(),
+});
+
export type Ping = z.infer<typeof tbBuildResponseList>;
export type Region = (typeof availableRegions)[number]; // TODO: rename type AvailabeRegion
export type Monitor = z.infer<typeof tbBuildMonitorList>;