Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
feat(status-page): fall back to manual mode when Tinybird is slow or errors (#2330)
Maximilian Kaskef1124d791dd3
diff --git a/packages/api/src/router/statusPage.ts b/packages/api/src/router/statusPage.tsindex 3d415a8b2..45bbf0b11 100644--- a/packages/api/src/router/statusPage.ts+++ b/packages/api/src/router/statusPage.ts@@ -38,6 +38,7 @@ import {getWorstVariant,isMonitorComponent,setDataByType,+ withTinybirdFallback,} from "./statusPage.utils";import {getMetricsLatencyMultiProcedure,@@ -660,17 +661,30 @@ export const statusPageRouter = createTRPCRouter({dns: getStatusProcedure("45d", "dns"),};- const [statusHttp, statusTcp, statusDns] = await Promise.all(- Object.entries(proceduresByType).map(([type, procedure]) => {- const monitorIds = monitorsByType[- type as keyof typeof proceduresByType- ].map((c) => c.monitor.id.toString());- if (monitorIds.length === 0) return null;- // NOTE: if manual mode, don't fetch data from tinybird- return input.barType === "manual" ? null : procedure({ monitorIds });- }),+ // Manual mode never touches Tinybird. Otherwise race the reads against+ // the fallback budget: a slow (>5s) or erroring Tinybird degrades the+ // whole page to manual mode so bars still render from DB events.+ const tinybird = await withTinybirdFallback(() =>+ input.barType === "manual"+ ? Promise.resolve([null, null, null])+ : Promise.all(+ Object.entries(proceduresByType).map(([type, procedure]) => {+ const monitorIds = monitorsByType[+ type as keyof typeof proceduresByType+ ].map((c) => c.monitor.id.toString());+ if (monitorIds.length === 0) return null;+ return procedure({ monitorIds });+ }),+ ),);+ const tinybirdUnhealthy = !tinybird.ok;+ const [statusHttp, statusTcp, statusDns] = tinybird.data ?? [+ null,+ null,+ null,+ ];+const statusDataByMonitorId = new Map<string,| Awaited<ReturnType<(typeof proceduresByType)["http"]>>["data"]@@ -713,6 +727,7 @@ export const statusPageRouter = createTRPCRouter({c.type === "monitor" &&c.monitor &&input.barType !== "manual" &&+ !tinybirdUnhealthy &&process.env.NOOP_UPTIME !== "true";let filledData: StatusData[];@@ -734,10 +749,11 @@ export const statusPageRouter = createTRPCRouter({});}- // Static components always use manual mode since they don't have real monitoring data- const effectiveBarType = c.type === "static" ? "manual" : input.barType;- const effectiveCardType =- c.type === "static" ? "manual" : input.cardType;+ // Static components have no monitoring data; a degraded Tinybird forces+ // every component into manual mode so bars/cards still render.+ const forceManual = c.type === "static" || tinybirdUnhealthy;+ const effectiveBarType = forceManual ? "manual" : input.barType;+ const effectiveCardType = forceManual ? "manual" : input.cardType;const processedData = setDataByType({events,@@ -951,19 +967,24 @@ export const statusPageRouter = createTRPCRouter({dns: getMetricsLatencyMultiProcedure("1d", "dns"),};+ // Slow/erroring Tinybird → empty latency data so the page still renders.+ const metrics = await withTinybirdFallback(() =>+ Promise.all(+ Object.entries(proceduresByType).map(([type, procedure]) => {+ const monitorIds = monitorsByType[+ type as keyof typeof proceduresByType+ ].map((c) => c.monitor.id.toString());+ if (monitorIds.length === 0) return null;+ return procedure({ monitorIds });+ }),+ ),+ );+const [metricsLatencyMultiHttp,metricsLatencyMultiTcp,metricsLatencyMultiDns,- ] = await Promise.all(- Object.entries(proceduresByType).map(([type, procedure]) => {- const monitorIds = monitorsByType[- type as keyof typeof proceduresByType- ].map((c) => c.monitor.id.toString());- if (monitorIds.length === 0) return null;- return procedure({ monitorIds });- }),- );+ ] = metrics.data ?? [null, null, null];const metricsDataByMonitorId = new Map<string,@@ -1069,24 +1090,33 @@ export const statusPageRouter = createTRPCRouter({const fromDate = startOfDay(subDays(new Date(), 7)).toISOString();const toDate = endOfDay(new Date()).toISOString();- const [latency, regions, uptime] = await Promise.all([- await proceduresByType[type].latency({- monitorId: _monitor.id.toString(),- fromDate,- toDate,- }),- await proceduresByType[type].regions({- monitorId: _monitor.id.toString(),- fromDate,- toDate,- }),- await proceduresByType[type].uptime({- monitorId: _monitor.id.toString(),- interval: 240,- fromDate,- toDate,- }),- ]);+ // Slow/erroring Tinybird → empty chart data so the page still renders.+ const metrics = await withTinybirdFallback(() =>+ Promise.all([+ proceduresByType[type].latency({+ monitorId: _monitor.id.toString(),+ fromDate,+ toDate,+ }),+ proceduresByType[type].regions({+ monitorId: _monitor.id.toString(),+ fromDate,+ toDate,+ }),+ proceduresByType[type].uptime({+ monitorId: _monitor.id.toString(),+ interval: 240,+ fromDate,+ toDate,+ }),+ ]),+ );++ const [latency, regions, uptime] = metrics.data ?? [+ { data: [] },+ { data: [] },+ { data: [] },+ ];return {...selectPublicMonitorSchema.parse(_monitor),diff --git a/packages/api/src/router/statusPage.utils.test.ts b/packages/api/src/router/statusPage.utils.test.tsindex 1961e85f9..f7fe16d98 100644--- a/packages/api/src/router/statusPage.utils.test.ts+++ b/packages/api/src/router/statusPage.utils.test.ts@@ -14,6 +14,7 @@ import {getEvents,getUptime,setDataByType,+ withTinybirdFallback,} from "./statusPage.utils";type StatusData = {@@ -2079,3 +2080,25 @@ describe("componentImpacts", () => {});});});++describe("withTinybirdFallback", () => {+ it("returns ok with data when the read resolves in time", async () => {+ const result = await withTinybirdFallback(async () => ({ data: [1, 2] }));+ expect(result).toEqual({ ok: true, data: { data: [1, 2] } });+ });++ it("falls back when the read throws", async () => {+ const result = await withTinybirdFallback(async () => {+ throw new Error("tinybird down");+ });+ expect(result).toEqual({ ok: false, data: null });+ });++ it("falls back when the read exceeds the timeout", async () => {+ const result = await withTinybirdFallback(+ () => new Promise((resolve) => setTimeout(resolve, 50)),+ 10,+ );+ expect(result).toEqual({ ok: false, data: null });+ });+});diff --git a/packages/api/src/router/statusPage.utils.ts b/packages/api/src/router/statusPage.utils.tsindex 6e7534f0a..5f6d5dfaa 100644--- a/packages/api/src/router/statusPage.utils.ts+++ b/packages/api/src/router/statusPage.utils.ts@@ -17,6 +17,43 @@ import {export * from "@openstatus/services/status-timeline";+// Status pages must render even when Tinybird is degraded. Read latency above+// this budget is treated as an outage and the page falls back to manual mode.+export const TINYBIRD_FALLBACK_TIMEOUT_MS = 5_000;++// Discriminated result of a guarded Tinybird read: `ok: true` carries the data,+// `ok: false` (timeout or error) carries `null` and signals manual-mode fallback.+export type TinybirdResult<T> =+ | { ok: true; data: T }+ | { ok: false; data: null };++// Races a Tinybird read against the fallback budget. `ok: false` (timeout or+// thrown error) signals the caller to serve manual mode — DB-authored events+// only — instead of hanging or 500ing on Tinybird.+export async function withTinybirdFallback<T>(+ fetch: () => Promise<T>,+ timeoutMs = TINYBIRD_FALLBACK_TIMEOUT_MS,+): Promise<TinybirdResult<T>> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ try {+ const data = await Promise.race([+ fetch(),+ new Promise<never>((_, reject) => {+ timer = setTimeout(+ () => reject(new Error("tinybird timeout")),+ timeoutMs,+ );+ }),+ ]);+ return { ok: true, data };+ } catch (err) {+ console.error("[status-page] tinybird unhealthy, using manual mode:", err);+ return { ok: false, data: null };+ } finally {+ clearTimeout(timer);+ }+}+type UptimeData = {day: string;events: Event[];