Skip to content

Open-source status page, incident response and uptime monitoring

feat(status-page): fall back to manual mode when Tinybird is slow or errors (#2330)

Maximilian Kaskef1124d791dd3
diff --git a/packages/api/src/router/statusPage.ts b/packages/api/src/router/statusPage.ts
index 3d415a8b2..45bbf0b11 100644
--- a/packages/api/src/router/statusPage.ts
+++ b/packages/api/src/router/statusPage.ts
@@ -38,6 +38,7 @@ import {
getWorstVariant,
isMonitorComponent,
setDataByType,
+ withTinybirdFallback,
} from "./statusPage.utils";
import {
getMetricsLatencyMultiProcedure,
@@ -660,17 +661,30 @@ export const statusPageRouter = createTRPCRouter({
dns: getStatusProcedure("45d", "dns"),
};
- const [statusHttp, statusTcp, statusDns] = await Promise.all(
- Object.entries(proceduresByType).map(([type, procedure]) => {
- const monitorIds = monitorsByType[
- type as keyof typeof proceduresByType
- ].map((c) => c.monitor.id.toString());
- if (monitorIds.length === 0) return null;
- // NOTE: if manual mode, don't fetch data from tinybird
- return input.barType === "manual" ? null : procedure({ monitorIds });
- }),
+ // Manual mode never touches Tinybird. Otherwise race the reads against
+ // the fallback budget: a slow (>5s) or erroring Tinybird degrades the
+ // whole page to manual mode so bars still render from DB events.
+ const tinybird = await withTinybirdFallback(() =>
+ input.barType === "manual"
+ ? Promise.resolve([null, null, null])
+ : Promise.all(
+ Object.entries(proceduresByType).map(([type, procedure]) => {
+ const monitorIds = monitorsByType[
+ type as keyof typeof proceduresByType
+ ].map((c) => c.monitor.id.toString());
+ if (monitorIds.length === 0) return null;
+ return procedure({ monitorIds });
+ }),
+ ),
);
+ const tinybirdUnhealthy = !tinybird.ok;
+ const [statusHttp, statusTcp, statusDns] = tinybird.data ?? [
+ null,
+ null,
+ null,
+ ];
+
const statusDataByMonitorId = new Map<
string,
| Awaited<ReturnType<(typeof proceduresByType)["http"]>>["data"]
@@ -713,6 +727,7 @@ export const statusPageRouter = createTRPCRouter({
c.type === "monitor" &&
c.monitor &&
input.barType !== "manual" &&
+ !tinybirdUnhealthy &&
process.env.NOOP_UPTIME !== "true";
let filledData: StatusData[];
@@ -734,10 +749,11 @@ export const statusPageRouter = createTRPCRouter({
});
}
- // Static components always use manual mode since they don't have real monitoring data
- const effectiveBarType = c.type === "static" ? "manual" : input.barType;
- const effectiveCardType =
- c.type === "static" ? "manual" : input.cardType;
+ // Static components have no monitoring data; a degraded Tinybird forces
+ // every component into manual mode so bars/cards still render.
+ const forceManual = c.type === "static" || tinybirdUnhealthy;
+ const effectiveBarType = forceManual ? "manual" : input.barType;
+ const effectiveCardType = forceManual ? "manual" : input.cardType;
const processedData = setDataByType({
events,
@@ -951,19 +967,24 @@ export const statusPageRouter = createTRPCRouter({
dns: getMetricsLatencyMultiProcedure("1d", "dns"),
};
+ // Slow/erroring Tinybird → empty latency data so the page still renders.
+ const metrics = await withTinybirdFallback(() =>
+ Promise.all(
+ Object.entries(proceduresByType).map(([type, procedure]) => {
+ const monitorIds = monitorsByType[
+ type as keyof typeof proceduresByType
+ ].map((c) => c.monitor.id.toString());
+ if (monitorIds.length === 0) return null;
+ return procedure({ monitorIds });
+ }),
+ ),
+ );
+
const [
metricsLatencyMultiHttp,
metricsLatencyMultiTcp,
metricsLatencyMultiDns,
- ] = await Promise.all(
- Object.entries(proceduresByType).map(([type, procedure]) => {
- const monitorIds = monitorsByType[
- type as keyof typeof proceduresByType
- ].map((c) => c.monitor.id.toString());
- if (monitorIds.length === 0) return null;
- return procedure({ monitorIds });
- }),
- );
+ ] = metrics.data ?? [null, null, null];
const metricsDataByMonitorId = new Map<
string,
@@ -1069,24 +1090,33 @@ export const statusPageRouter = createTRPCRouter({
const fromDate = startOfDay(subDays(new Date(), 7)).toISOString();
const toDate = endOfDay(new Date()).toISOString();
- const [latency, regions, uptime] = await Promise.all([
- await proceduresByType[type].latency({
- monitorId: _monitor.id.toString(),
- fromDate,
- toDate,
- }),
- await proceduresByType[type].regions({
- monitorId: _monitor.id.toString(),
- fromDate,
- toDate,
- }),
- await proceduresByType[type].uptime({
- monitorId: _monitor.id.toString(),
- interval: 240,
- fromDate,
- toDate,
- }),
- ]);
+ // Slow/erroring Tinybird → empty chart data so the page still renders.
+ const metrics = await withTinybirdFallback(() =>
+ Promise.all([
+ proceduresByType[type].latency({
+ monitorId: _monitor.id.toString(),
+ fromDate,
+ toDate,
+ }),
+ proceduresByType[type].regions({
+ monitorId: _monitor.id.toString(),
+ fromDate,
+ toDate,
+ }),
+ proceduresByType[type].uptime({
+ monitorId: _monitor.id.toString(),
+ interval: 240,
+ fromDate,
+ toDate,
+ }),
+ ]),
+ );
+
+ const [latency, regions, uptime] = metrics.data ?? [
+ { data: [] },
+ { data: [] },
+ { data: [] },
+ ];
return {
...selectPublicMonitorSchema.parse(_monitor),
diff --git a/packages/api/src/router/statusPage.utils.test.ts b/packages/api/src/router/statusPage.utils.test.ts
index 1961e85f9..f7fe16d98 100644
--- a/packages/api/src/router/statusPage.utils.test.ts
+++ b/packages/api/src/router/statusPage.utils.test.ts
@@ -14,6 +14,7 @@ import {
getEvents,
getUptime,
setDataByType,
+ withTinybirdFallback,
} from "./statusPage.utils";
type StatusData = {
@@ -2079,3 +2080,25 @@ describe("componentImpacts", () => {
});
});
});
+
+describe("withTinybirdFallback", () => {
+ it("returns ok with data when the read resolves in time", async () => {
+ const result = await withTinybirdFallback(async () => ({ data: [1, 2] }));
+ expect(result).toEqual({ ok: true, data: { data: [1, 2] } });
+ });
+
+ it("falls back when the read throws", async () => {
+ const result = await withTinybirdFallback(async () => {
+ throw new Error("tinybird down");
+ });
+ expect(result).toEqual({ ok: false, data: null });
+ });
+
+ it("falls back when the read exceeds the timeout", async () => {
+ const result = await withTinybirdFallback(
+ () => new Promise((resolve) => setTimeout(resolve, 50)),
+ 10,
+ );
+ expect(result).toEqual({ ok: false, data: null });
+ });
+});
diff --git a/packages/api/src/router/statusPage.utils.ts b/packages/api/src/router/statusPage.utils.ts
index 6e7534f0a..5f6d5dfaa 100644
--- a/packages/api/src/router/statusPage.utils.ts
+++ b/packages/api/src/router/statusPage.utils.ts
@@ -17,6 +17,43 @@ import {
export * from "@openstatus/services/status-timeline";
+// Status pages must render even when Tinybird is degraded. Read latency above
+// this budget is treated as an outage and the page falls back to manual mode.
+export const TINYBIRD_FALLBACK_TIMEOUT_MS = 5_000;
+
+// Discriminated result of a guarded Tinybird read: `ok: true` carries the data,
+// `ok: false` (timeout or error) carries `null` and signals manual-mode fallback.
+export type TinybirdResult<T> =
+ | { ok: true; data: T }
+ | { ok: false; data: null };
+
+// Races a Tinybird read against the fallback budget. `ok: false` (timeout or
+// thrown error) signals the caller to serve manual mode — DB-authored events
+// only — instead of hanging or 500ing on Tinybird.
+export async function withTinybirdFallback<T>(
+ fetch: () => Promise<T>,
+ timeoutMs = TINYBIRD_FALLBACK_TIMEOUT_MS,
+): Promise<TinybirdResult<T>> {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ const data = await Promise.race([
+ fetch(),
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error("tinybird timeout")),
+ timeoutMs,
+ );
+ }),
+ ]);
+ return { ok: true, data };
+ } catch (err) {
+ console.error("[status-page] tinybird unhealthy, using manual mode:", err);
+ return { ok: false, data: null };
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
type UptimeData = {
day: string;
events: Event[];