Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
Feat/tb cost reduction (#381)
Maximilian Kaskefedb7eda272b
diff --git a/apps/web/src/app/api/checker/regions/_checker.ts b/apps/web/src/app/api/checker/regions/_checker.tsindex 6f840b5e1..e854e0372 100644--- a/apps/web/src/app/api/checker/regions/_checker.ts+++ b/apps/web/src/app/api/checker/regions/_checker.ts@@ -23,40 +23,24 @@ export const monitorSchema = tbIngestPingResponse.pick({const tb = new Tinybird({ token: env.TINY_BIRD_API_KEY });const monitor = async (- res: { text: () => Promise<string>; status: number },- monitorInfo: z.infer<typeof payloadSchema>,+ payload: Payload,+ statusCode: number,region: string,latency: number,) => {- const text = (await res.text()) || "";- if (monitorInfo.pageIds.length > 0) {- for (const pageId of monitorInfo.pageIds) {- const { pageIds, ...rest } = monitorInfo;- await publishPingResponse(tb)({- ...rest,- id: nanoid(), // TBD: we don't need it- pageId: pageId,- timestamp: Date.now(),- statusCode: res.status,- latency,- region,- metadata: JSON.stringify({ text }),- });- }- } else {- const { pageIds, ...rest } = monitorInfo;-- await publishPingResponse(tb)({- ...rest,- id: nanoid(), // TBD: we don't need it- pageId: "",- timestamp: Date.now(),- statusCode: res.status,- latency,- region,- metadata: JSON.stringify({ text }),- });- }+ const { monitorId, cronTimestamp, url, workspaceId } = payload;++ await publishPingResponse(tb)({+ id: nanoid(), // TBD: we don't need it+ timestamp: Date.now(),+ statusCode,+ monitorId,+ cronTimestamp,+ url,+ workspaceId,+ latency,+ region,+ });};export const checker = async (request: Request, region: string) => {@@ -87,7 +71,7 @@ export const checker = async (request: Request, region: string) => {const res = await ping(result.data);const endTime = performance.now();const latency = endTime - startTime;- await monitor(res, result.data, region, latency);+ await monitor(result.data, res.status, region, latency);if (res.ok) {if (result.data?.status === "error") {await updateMonitorStatus({@@ -109,12 +93,7 @@ export const checker = async (request: Request, region: string) => {console.error(e);// if on the third retry we still get an error, we should report itif (request.headers.get("Upstash-Retried") === "2") {- await monitor(- { status: 500, text: () => Promise.resolve(`${e}`) },- result.data,- region,- -1,- );+ await monitor(result.data, 500, region, -1);if (result.data?.status !== "error") {await triggerAlerting({ monitorId: result.data.monitorId });await updateMonitorStatus({diff --git a/apps/web/src/app/api/og/route.tsx b/apps/web/src/app/api/og/route.tsxindex ff207b120..dd22f4ac6 100644--- a/apps/web/src/app/api/og/route.tsx+++ b/apps/web/src/app/api/og/route.tsx@@ -46,7 +46,6 @@ export async function GET(req: Request) {(monitorId &&(await getMonitorListData({monitorId,- groupBy: "day",limit: LIMIT,}))) ||[];diff --git a/apps/web/src/app/play/page.tsx b/apps/web/src/app/play/page.tsxindex cfbd6a417..cff7f767d 100644--- a/apps/web/src/app/play/page.tsx+++ b/apps/web/src/app/play/page.tsx@@ -1,35 +1,10 @@-import * as z from "zod";--import { groupByRange } from "@openstatus/tinybird";-import { Tracker } from "@/components/tracker";import { getMonitorListData } from "@/lib/tb";-import { ToggleButton } from "./toggle-button";--/**- * allowed URL search params- */-const searchParamsSchema = z.object({- groupBy: z.enum(groupByRange).optional(),-});--export default async function PlayPage({- searchParams,-}: {- searchParams: { [key: string]: string | string[] | undefined };-}) {- const search = searchParamsSchema.safeParse(searchParams);- const params = search.success ? search.data : undefined;-- const data = search.success- ? await getMonitorListData({ monitorId: "openstatusPing", ...params })- : await getMonitorListData({ monitorId: "openstatusPing" });+export default async function PlayPage() {+ const data = await getMonitorListData({ monitorId: "openstatusPing" });return (<div className="relative flex flex-col items-center justify-center gap-4">- <div className="absolute right-2 top-2">- <ToggleButton groupBy={params?.groupBy} />- </div><p className="font-cal mb-1 text-3xl">Status</p><p className="text-muted-foreground text-lg font-light">Build your own within seconds.diff --git a/apps/web/src/app/play/toggle-button.tsx b/apps/web/src/app/play/toggle-button.tsxdeleted file mode 100644index d0cc61a46..000000000--- a/apps/web/src/app/play/toggle-button.tsx+++ /dev/null@@ -1,48 +0,0 @@-"use client";--import * as React from "react";-import { usePathname, useRouter } from "next/navigation";-import { CalendarDays } from "lucide-react";--import type { GroupByRange } from "@openstatus/tinybird";-import {- Button,- Tooltip,- TooltipContent,- TooltipProvider,- TooltipTrigger,-} from "@openstatus/ui";--import useUpdateSearchParams from "@/hooks/use-update-search-params";--// TODO: move to @/components folder-export function ToggleButton({ groupBy }: { groupBy?: GroupByRange }) {- const router = useRouter();- const updateSearchParams = useUpdateSearchParams();- const pathname = usePathname();-- const isDay = groupBy === "day";-- return (- <TooltipProvider>- <Tooltip>- <TooltipTrigger asChild>- <Button- variant={isDay ? "default" : "outline"}- size="icon"- onClick={() => {- router.replace(- `${pathname}?${updateSearchParams({- groupBy: isDay ? null : "day",- })}`,- );- }}- >- <CalendarDays className="h-4 w-4" />- </Button>- </TooltipTrigger>- <TooltipContent>{isDay ? "by ping" : "by day"}</TooltipContent>- </Tooltip>- </TooltipProvider>- );-}diff --git a/apps/web/src/components/data-table/columns.tsx b/apps/web/src/components/data-table/columns.tsxindex 087bbbab0..aa92fb825 100644--- a/apps/web/src/components/data-table/columns.tsx+++ b/apps/web/src/components/data-table/columns.tsx@@ -7,7 +7,6 @@ import type { Ping } from "@openstatus/tinybird";import { regionsDict } from "@/data/regions-dictionary";import { DataTableColumnHeader } from "./data-table-column-header";-import { DataTableRowActions } from "./data-table-row-action";import { DataTableStatusBadge } from "./data-table-status-badge";export const columns: ColumnDef<Ping>[] = [@@ -63,15 +62,4 @@ export const columns: ColumnDef<Ping>[] = [return value.includes(row.getValue(id));},},-- {- id: "actions",- cell: ({ row }) => {- return (- <div className="text-right">- <DataTableRowActions row={row} />- </div>- );- },- },];diff --git a/apps/web/src/components/status-page/monitor.tsx b/apps/web/src/components/status-page/monitor.tsxindex 26482bfec..a543d939d 100644--- a/apps/web/src/components/status-page/monitor.tsx+++ b/apps/web/src/components/status-page/monitor.tsx@@ -10,10 +10,7 @@ export const Monitor = async ({}: {monitor: z.infer<typeof selectPublicMonitorSchema>;}) => {- const data = await getMonitorListData({- monitorId: String(monitor.id),- groupBy: "day",- });+ const data = await getMonitorListData({ monitorId: String(monitor.id) });if (!data) return <div>Something went wrong</div>;return (<Trackerdiff --git a/apps/web/src/lib/tb.ts b/apps/web/src/lib/tb.tsindex d101e2bf4..d64c2606a 100644--- a/apps/web/src/lib/tb.ts+++ b/apps/web/src/lib/tb.ts@@ -26,7 +26,7 @@ export async function getResponseListData(props: Partial<ResponseListParams>) {return;}-export async function getMonitorListData(props: Partial<MonitorListParams>) {+export async function getMonitorListData(props: MonitorListParams) {try {const res = await getMonitorList(tb)(props);return res.data;diff --git a/packages/tinybird/datasources/aggregated_monitor_per_day_mv.datasource b/packages/tinybird/datasources/aggregated_monitor_per_day_mv.datasourcenew file mode 100644index 000000000..5f018bc5a--- /dev/null+++ b/packages/tinybird/datasources/aggregated_monitor_per_day_mv.datasource@@ -0,0 +1,12 @@+# Data Source created from Pipe 'aggregated_monitor_day_mv'+VERSION 0++SCHEMA >+ `day` Int64,+ `monitorId` String,+ `avgLatency` AggregateFunction(avg, Int16),+ `count` AggregateFunction(count),+ `ok` AggregateFunction(count, Nullable(UInt8))++ENGINE "AggregatingMergeTree"+ENGINE_SORTING_KEY "day, monitorId"diff --git a/packages/tinybird/datasources/ping_response.datasource b/packages/tinybird/datasources/ping_response.datasourceindex bc042d8aa..ec40d58bc 100644--- a/packages/tinybird/datasources/ping_response.datasource+++ b/packages/tinybird/datasources/ping_response.datasource@@ -1,17 +1,15 @@-VERSION 3+VERSION 4SCHEMA >- `id` String `json:$.id`,- `latency` Int16 `json:$.latency`,- `monitorId` String `json:$.monitorId`,- `pageId` String `json:$.pageId`,- `region` LowCardinality(String) `json:$.region`,- `statusCode` Int16 `json:$.statusCode`,- `timestamp` Int64 `json:$.timestamp`,- `url` String `json:$.url`,- `workspaceId` String `json:$.workspaceId`,- `cronTimestamp` Int64 `json:$.cronTimestamp`,- `metadata` String `json:$.metadata`+ `id` String,+ `latency` Int16,+ `monitorId` String,+ `region` LowCardinality(String),+ `statusCode` Int16,+ `timestamp` Int64,+ `url` String,+ `workspaceId` String,+ `cronTimestamp` Int64ENGINE "MergeTree"ENGINE_SORTING_KEY "monitorId, cronTimestamp"diff --git a/packages/tinybird/pipes/aggregated_monitor_day_mv.pipe b/packages/tinybird/pipes/aggregated_monitor_day_mv.pipenew file mode 100644index 000000000..21c0ad324--- /dev/null+++ b/packages/tinybird/pipes/aggregated_monitor_day_mv.pipe@@ -0,0 +1,23 @@+VERSION 0++NODE aggregated_monitor_day_0+SQL >++ SELECT+ toUnixTimestamp64Milli(toDateTime64(DATE(cronTimestamp / 1000), 3)) AS day,+ monitorId,+ avgState(latency) AS avgLatency,+ countState() AS count,+ countState(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok+ FROM ping_response__v3+ WHERE (day IS NOT NULL) AND (day != 0)+ GROUP BY+ day,+ monitorId+ ORDER BY day DESC++TYPE materialized+DATASOURCE aggregated_monitor_per_day_0_mv+ENGINE "AggregatingMergeTree"+ENGINE_SORTING_KEY "day, monitorId"+diff --git a/packages/tinybird/pipes/home_stats.pipe b/packages/tinybird/pipes/home_stats.pipeindex ffd9e95b3..85c2c0fcc 100644--- a/packages/tinybird/pipes/home_stats.pipe+++ b/packages/tinybird/pipes/home_stats.pipe@@ -1,6 +1,6 @@VERSION 0-NODE home_stats__0+NODE home_stats_0SQL >%diff --git a/packages/tinybird/pipes/monitor_list.pipe b/packages/tinybird/pipes/monitor_list.pipeindex 6fac82f4d..5cd36e7de 100644--- a/packages/tinybird/pipes/monitor_list.pipe+++ b/packages/tinybird/pipes/monitor_list.pipe@@ -1,20 +1,14 @@-VERSION 0+VERSION 1NODE monitor_list_0SQL >%- SELECT- {% if defined(groupBy) and groupBy == "day" %}- toUnixTimestamp64Milli(toDateTime64(DATE(cronTimestamp / 1000), 3))- {% else %}- cronTimestamp- {% end %}- AS cronTimestamp, ROUND(AVG(latency)) as avgLatency, COUNT(*) as count, COUNT(CASE WHEN statusCode BETWEEN 200 AND 299 THEN 1 ELSE NULL END) as ok- FROM ping_response- WHERE monitorId = {{ String(monitorId, 'openstatusPing') }} AND cronTimestamp IS NOT NULL- GROUP BY cronTimestamp+ SELECT day as cronTimestamp, round(avgMerge(avgLatency)) avgLatency, countMerge(ok) ok, countMerge(count) count+ FROM aggregated_monitor_per_day_mv+ WHERE monitorId = {{ String(monitorId, 'openstatusPing') }}+ GROUP BY day, monitorIdORDER BY cronTimestamp DESC- LIMIT {{Int32(limit, 1000)}}+ LIMIT {{Int32(limit, 100)}}diff --git a/packages/tinybird/pipes/response_list.pipe b/packages/tinybird/pipes/response_list.pipeindex 4309c7859..593c2c35f 100644--- a/packages/tinybird/pipes/response_list.pipe+++ b/packages/tinybird/pipes/response_list.pipe@@ -1,10 +1,10 @@-VERSION 0+VERSION 1NODE response_list_0SQL >%- SELECT *+ SELECT id, latency, monitorId, region, statusCode, timestamp, url, workspaceId, cronTimestampFROM ping_responseWHERE monitorId = {{ String(monitorId, 'openstatusPing') }}{% if defined(region) %}diff --git a/packages/tinybird/src/client.ts b/packages/tinybird/src/client.tsindex 0039755b9..5ae1d2c46 100644--- a/packages/tinybird/src/client.ts+++ b/packages/tinybird/src/client.ts@@ -15,14 +15,14 @@ import {export function publishPingResponse(tb: Tinybird) {return tb.buildIngestEndpoint({- datasource: "ping_response__v3",+ datasource: "ping_response__v4",event: tbIngestPingResponse,});}export function getResponseList(tb: Tinybird) {return tb.buildPipe({- pipe: "response_list__v0",+ pipe: "response_list__v1",parameters: tbParameterResponseList,data: tbBuildResponseList,opts: {@@ -33,7 +33,7 @@ export function getResponseList(tb: Tinybird) {export function getMonitorList(tb: Tinybird) {return tb.buildPipe({- pipe: "monitor_list__v0",+ pipe: "monitor_list__v1",parameters: tbParameterMonitorList,data: tbBuildMonitorList,opts: {@@ -49,7 +49,7 @@ export function getMonitorList(tb: Tinybird) {*/export function getHomeMonitorList(tb: Tinybird) {return tb.buildPipe({- pipe: "monitor_list__v0",+ pipe: "monitor_list__v1",parameters: tbParameterMonitorList,data: tbBuildMonitorList,opts: {diff --git a/packages/tinybird/src/validation.ts b/packages/tinybird/src/validation.tsindex f40094ca4..32a9598cd 100644--- a/packages/tinybird/src/validation.ts+++ b/packages/tinybird/src/validation.ts@@ -25,45 +25,37 @@ export const availableRegions = [] as const;/**- * Values for the datasource ping_response__v3+ * Values for the datasource ping_response__v4*/export const tbIngestPingResponse = z.object({id: z.string(),workspaceId: z.string(),- pageId: z.string(),monitorId: z.string(),timestamp: z.number().int(),statusCode: z.number().int(),latency: z.number().int(), // in mscronTimestamp: z.number().int().optional().nullable().default(Date.now()),url: z.string().url(),- metadata: z.string().optional().default("{}").nullable(),- region: z.string().min(4).max(4),+ region: z.string().min(4).max(4), // REMINDER: won't work on fy});/**- * Values from the pip response_list__v0+ * Values from the pipe response_list__v1*/export const tbBuildResponseList = z.object({id: z.string(),workspaceId: z.string(),- pageId: z.string(),monitorId: z.string(),timestamp: z.number().int(),statusCode: z.number().int(),latency: z.number().int(), // in mscronTimestamp: z.number().int().nullable().default(Date.now()),url: z.string().url(),- metadata: z- .string()- .default("{}")- .transform((t) => JSON.parse(t))- .nullable(),region: z.enum(availableRegions),});/**- * Params for pipe response_list__v0+ * Params for pipe response_list__v1*/export const tbParameterResponseList = z.object({monitorId: z.string().default(""), // REMINDER: remove default once alpha@@ -75,24 +67,15 @@ export const tbParameterResponseList = z.object({});/**- * All `groupBy` options for the monitoring list- * - "day" will aggregate data within the same day- * - "cron" will get data from single cron job- */-export const groupByRange = ["day", "cron"] as const;--/**- * Params for pipe monitor_list__v0+ * Params for pipe monitor_list__v1*/export const tbParameterMonitorList = z.object({- monitorId: z.string().optional().default(""), // REMINDER: remove default once alpha- limit: z.number().int().optional().default(2500), // one day has 2448 pings (17 (regions) * 6 (per hour) * 24)- cronTimestamp: z.number().int().optional(),- groupBy: z.enum(groupByRange).optional(), // TODO: rename to frequency: z.enum(["1d", "auto"]) - where "auto" the default periodicity setup+ monitorId: z.string(),+ limit: z.number().int().default(60).optional(), // 40 days});/**- * Values from the pipe monitor_list__v0+ * Values from the pipe monitor_list__v1*/export const tbBuildMonitorList = z.object({count: z.number().int(),@@ -123,4 +106,3 @@ export type HomeStats = z.infer<typeof tbBuildHomeStats>;export type ResponseListParams = z.infer<typeof tbParameterResponseList>;export type MonitorListParams = z.infer<typeof tbParameterMonitorList>;export type HomeStatsParams = z.infer<typeof tbParameterHomeStats>;-export type GroupByRange = (typeof groupByRange)[number];