Skip to content

Open-source status page, incident response and uptime monitoring

Feat/tb cost reduction (#381)

Maximilian Kaskefedb7eda272b
diff --git a/apps/web/src/app/api/checker/regions/_checker.ts b/apps/web/src/app/api/checker/regions/_checker.ts
index 6f840b5e1..e854e0372 100644
--- a/apps/web/src/app/api/checker/regions/_checker.ts
+++ b/apps/web/src/app/api/checker/regions/_checker.ts
@@ -23,40 +23,24 @@ export const monitorSchema = tbIngestPingResponse.pick({
const tb = new Tinybird({ token: env.TINY_BIRD_API_KEY });
const monitor = async (
- res: { text: () => Promise<string>; status: number },
- monitorInfo: z.infer<typeof payloadSchema>,
+ payload: Payload,
+ statusCode: number,
region: string,
latency: number,
) => {
- const text = (await res.text()) || "";
- if (monitorInfo.pageIds.length > 0) {
- for (const pageId of monitorInfo.pageIds) {
- const { pageIds, ...rest } = monitorInfo;
- await publishPingResponse(tb)({
- ...rest,
- id: nanoid(), // TBD: we don't need it
- pageId: pageId,
- timestamp: Date.now(),
- statusCode: res.status,
- latency,
- region,
- metadata: JSON.stringify({ text }),
- });
- }
- } else {
- const { pageIds, ...rest } = monitorInfo;
-
- await publishPingResponse(tb)({
- ...rest,
- id: nanoid(), // TBD: we don't need it
- pageId: "",
- timestamp: Date.now(),
- statusCode: res.status,
- latency,
- region,
- metadata: JSON.stringify({ text }),
- });
- }
+ const { monitorId, cronTimestamp, url, workspaceId } = payload;
+
+ await publishPingResponse(tb)({
+ id: nanoid(), // TBD: we don't need it
+ timestamp: Date.now(),
+ statusCode,
+ monitorId,
+ cronTimestamp,
+ url,
+ workspaceId,
+ latency,
+ region,
+ });
};
export const checker = async (request: Request, region: string) => {
@@ -87,7 +71,7 @@ export const checker = async (request: Request, region: string) => {
const res = await ping(result.data);
const endTime = performance.now();
const latency = endTime - startTime;
- await monitor(res, result.data, region, latency);
+ await monitor(result.data, res.status, region, latency);
if (res.ok) {
if (result.data?.status === "error") {
await updateMonitorStatus({
@@ -109,12 +93,7 @@ export const checker = async (request: Request, region: string) => {
console.error(e);
// if on the third retry we still get an error, we should report it
if (request.headers.get("Upstash-Retried") === "2") {
- await monitor(
- { status: 500, text: () => Promise.resolve(`${e}`) },
- result.data,
- region,
- -1,
- );
+ await monitor(result.data, 500, region, -1);
if (result.data?.status !== "error") {
await triggerAlerting({ monitorId: result.data.monitorId });
await updateMonitorStatus({
diff --git a/apps/web/src/app/api/og/route.tsx b/apps/web/src/app/api/og/route.tsx
index ff207b120..dd22f4ac6 100644
--- a/apps/web/src/app/api/og/route.tsx
+++ b/apps/web/src/app/api/og/route.tsx
@@ -46,7 +46,6 @@ export async function GET(req: Request) {
(monitorId &&
(await getMonitorListData({
monitorId,
- groupBy: "day",
limit: LIMIT,
}))) ||
[];
diff --git a/apps/web/src/app/play/page.tsx b/apps/web/src/app/play/page.tsx
index cfbd6a417..cff7f767d 100644
--- a/apps/web/src/app/play/page.tsx
+++ b/apps/web/src/app/play/page.tsx
@@ -1,35 +1,10 @@
-import * as z from "zod";
-
-import { groupByRange } from "@openstatus/tinybird";
-
import { Tracker } from "@/components/tracker";
import { getMonitorListData } from "@/lib/tb";
-import { ToggleButton } from "./toggle-button";
-
-/**
- * allowed URL search params
- */
-const searchParamsSchema = z.object({
- groupBy: z.enum(groupByRange).optional(),
-});
-
-export default async function PlayPage({
- searchParams,
-}: {
- searchParams: { [key: string]: string | string[] | undefined };
-}) {
- const search = searchParamsSchema.safeParse(searchParams);
- const params = search.success ? search.data : undefined;
-
- const data = search.success
- ? await getMonitorListData({ monitorId: "openstatusPing", ...params })
- : await getMonitorListData({ monitorId: "openstatusPing" });
+export default async function PlayPage() {
+ const data = await getMonitorListData({ monitorId: "openstatusPing" });
return (
<div className="relative flex flex-col items-center justify-center gap-4">
- <div className="absolute right-2 top-2">
- <ToggleButton groupBy={params?.groupBy} />
- </div>
<p className="font-cal mb-1 text-3xl">Status</p>
<p className="text-muted-foreground text-lg font-light">
Build your own within seconds.
diff --git a/apps/web/src/app/play/toggle-button.tsx b/apps/web/src/app/play/toggle-button.tsx
deleted file mode 100644
index d0cc61a46..000000000
--- a/apps/web/src/app/play/toggle-button.tsx
+++ /dev/null
@@ -1,48 +0,0 @@
-"use client";
-
-import * as React from "react";
-import { usePathname, useRouter } from "next/navigation";
-import { CalendarDays } from "lucide-react";
-
-import type { GroupByRange } from "@openstatus/tinybird";
-import {
- Button,
- Tooltip,
- TooltipContent,
- TooltipProvider,
- TooltipTrigger,
-} from "@openstatus/ui";
-
-import useUpdateSearchParams from "@/hooks/use-update-search-params";
-
-// TODO: move to @/components folder
-export function ToggleButton({ groupBy }: { groupBy?: GroupByRange }) {
- const router = useRouter();
- const updateSearchParams = useUpdateSearchParams();
- const pathname = usePathname();
-
- const isDay = groupBy === "day";
-
- return (
- <TooltipProvider>
- <Tooltip>
- <TooltipTrigger asChild>
- <Button
- variant={isDay ? "default" : "outline"}
- size="icon"
- onClick={() => {
- router.replace(
- `${pathname}?${updateSearchParams({
- groupBy: isDay ? null : "day",
- })}`,
- );
- }}
- >
- <CalendarDays className="h-4 w-4" />
- </Button>
- </TooltipTrigger>
- <TooltipContent>{isDay ? "by ping" : "by day"}</TooltipContent>
- </Tooltip>
- </TooltipProvider>
- );
-}
diff --git a/apps/web/src/components/data-table/columns.tsx b/apps/web/src/components/data-table/columns.tsx
index 087bbbab0..aa92fb825 100644
--- a/apps/web/src/components/data-table/columns.tsx
+++ b/apps/web/src/components/data-table/columns.tsx
@@ -7,7 +7,6 @@ import type { Ping } from "@openstatus/tinybird";
import { regionsDict } from "@/data/regions-dictionary";
import { DataTableColumnHeader } from "./data-table-column-header";
-import { DataTableRowActions } from "./data-table-row-action";
import { DataTableStatusBadge } from "./data-table-status-badge";
export const columns: ColumnDef<Ping>[] = [
@@ -63,15 +62,4 @@ export const columns: ColumnDef<Ping>[] = [
return value.includes(row.getValue(id));
},
},
-
- {
- id: "actions",
- cell: ({ row }) => {
- return (
- <div className="text-right">
- <DataTableRowActions row={row} />
- </div>
- );
- },
- },
];
diff --git a/apps/web/src/components/status-page/monitor.tsx b/apps/web/src/components/status-page/monitor.tsx
index 26482bfec..a543d939d 100644
--- a/apps/web/src/components/status-page/monitor.tsx
+++ b/apps/web/src/components/status-page/monitor.tsx
@@ -10,10 +10,7 @@ export const Monitor = async ({
}: {
monitor: z.infer<typeof selectPublicMonitorSchema>;
}) => {
- const data = await getMonitorListData({
- monitorId: String(monitor.id),
- groupBy: "day",
- });
+ const data = await getMonitorListData({ monitorId: String(monitor.id) });
if (!data) return <div>Something went wrong</div>;
return (
<Tracker
diff --git a/apps/web/src/lib/tb.ts b/apps/web/src/lib/tb.ts
index d101e2bf4..d64c2606a 100644
--- a/apps/web/src/lib/tb.ts
+++ b/apps/web/src/lib/tb.ts
@@ -26,7 +26,7 @@ export async function getResponseListData(props: Partial<ResponseListParams>) {
return;
}
-export async function getMonitorListData(props: Partial<MonitorListParams>) {
+export async function getMonitorListData(props: MonitorListParams) {
try {
const res = await getMonitorList(tb)(props);
return res.data;
diff --git a/packages/tinybird/datasources/aggregated_monitor_per_day_mv.datasource b/packages/tinybird/datasources/aggregated_monitor_per_day_mv.datasource
new file mode 100644
index 000000000..5f018bc5a
--- /dev/null
+++ b/packages/tinybird/datasources/aggregated_monitor_per_day_mv.datasource
@@ -0,0 +1,12 @@
+# Data Source created from Pipe 'aggregated_monitor_day_mv'
+VERSION 0
+
+SCHEMA >
+ `day` Int64,
+ `monitorId` String,
+ `avgLatency` AggregateFunction(avg, Int16),
+ `count` AggregateFunction(count),
+ `ok` AggregateFunction(count, Nullable(UInt8))
+
+ENGINE "AggregatingMergeTree"
+ENGINE_SORTING_KEY "day, monitorId"
diff --git a/packages/tinybird/datasources/ping_response.datasource b/packages/tinybird/datasources/ping_response.datasource
index bc042d8aa..ec40d58bc 100644
--- a/packages/tinybird/datasources/ping_response.datasource
+++ b/packages/tinybird/datasources/ping_response.datasource
@@ -1,17 +1,15 @@
-VERSION 3
+VERSION 4
SCHEMA >
- `id` String `json:$.id`,
- `latency` Int16 `json:$.latency`,
- `monitorId` String `json:$.monitorId`,
- `pageId` String `json:$.pageId`,
- `region` LowCardinality(String) `json:$.region`,
- `statusCode` Int16 `json:$.statusCode`,
- `timestamp` Int64 `json:$.timestamp`,
- `url` String `json:$.url`,
- `workspaceId` String `json:$.workspaceId`,
- `cronTimestamp` Int64 `json:$.cronTimestamp`,
- `metadata` String `json:$.metadata`
+ `id` String,
+ `latency` Int16,
+ `monitorId` String,
+ `region` LowCardinality(String),
+ `statusCode` Int16,
+ `timestamp` Int64,
+ `url` String,
+ `workspaceId` String,
+ `cronTimestamp` Int64
ENGINE "MergeTree"
ENGINE_SORTING_KEY "monitorId, cronTimestamp"
diff --git a/packages/tinybird/pipes/aggregated_monitor_day_mv.pipe b/packages/tinybird/pipes/aggregated_monitor_day_mv.pipe
new file mode 100644
index 000000000..21c0ad324
--- /dev/null
+++ b/packages/tinybird/pipes/aggregated_monitor_day_mv.pipe
@@ -0,0 +1,23 @@
+VERSION 0
+
+NODE aggregated_monitor_day_0
+SQL >
+
+ SELECT
+ toUnixTimestamp64Milli(toDateTime64(DATE(cronTimestamp / 1000), 3)) AS day,
+ monitorId,
+ avgState(latency) AS avgLatency,
+ countState() AS count,
+ countState(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok
+ FROM ping_response__v3
+ WHERE (day IS NOT NULL) AND (day != 0)
+ GROUP BY
+ day,
+ monitorId
+ ORDER BY day DESC
+
+TYPE materialized
+DATASOURCE aggregated_monitor_per_day_0_mv
+ENGINE "AggregatingMergeTree"
+ENGINE_SORTING_KEY "day, monitorId"
+
diff --git a/packages/tinybird/pipes/home_stats.pipe b/packages/tinybird/pipes/home_stats.pipe
index ffd9e95b3..85c2c0fcc 100644
--- a/packages/tinybird/pipes/home_stats.pipe
+++ b/packages/tinybird/pipes/home_stats.pipe
@@ -1,6 +1,6 @@
VERSION 0
-NODE home_stats__0
+NODE home_stats_0
SQL >
%
diff --git a/packages/tinybird/pipes/monitor_list.pipe b/packages/tinybird/pipes/monitor_list.pipe
index 6fac82f4d..5cd36e7de 100644
--- a/packages/tinybird/pipes/monitor_list.pipe
+++ b/packages/tinybird/pipes/monitor_list.pipe
@@ -1,20 +1,14 @@
-VERSION 0
+VERSION 1
NODE monitor_list_0
SQL >
%
- SELECT
- {% if defined(groupBy) and groupBy == "day" %}
- toUnixTimestamp64Milli(toDateTime64(DATE(cronTimestamp / 1000), 3))
- {% else %}
- cronTimestamp
- {% end %}
- AS cronTimestamp, ROUND(AVG(latency)) as avgLatency, COUNT(*) as count, COUNT(CASE WHEN statusCode BETWEEN 200 AND 299 THEN 1 ELSE NULL END) as ok
- FROM ping_response
- WHERE monitorId = {{ String(monitorId, 'openstatusPing') }} AND cronTimestamp IS NOT NULL
- GROUP BY cronTimestamp
+ SELECT day as cronTimestamp, round(avgMerge(avgLatency)) avgLatency, countMerge(ok) ok, countMerge(count) count
+ FROM aggregated_monitor_per_day_mv
+ WHERE monitorId = {{ String(monitorId, 'openstatusPing') }}
+ GROUP BY day, monitorId
ORDER BY cronTimestamp DESC
- LIMIT {{Int32(limit, 1000)}}
+ LIMIT {{Int32(limit, 100)}}
diff --git a/packages/tinybird/pipes/response_list.pipe b/packages/tinybird/pipes/response_list.pipe
index 4309c7859..593c2c35f 100644
--- a/packages/tinybird/pipes/response_list.pipe
+++ b/packages/tinybird/pipes/response_list.pipe
@@ -1,10 +1,10 @@
-VERSION 0
+VERSION 1
NODE response_list_0
SQL >
%
- SELECT *
+ SELECT id, latency, monitorId, region, statusCode, timestamp, url, workspaceId, cronTimestamp
FROM ping_response
WHERE monitorId = {{ String(monitorId, 'openstatusPing') }}
{% if defined(region) %}
diff --git a/packages/tinybird/src/client.ts b/packages/tinybird/src/client.ts
index 0039755b9..5ae1d2c46 100644
--- a/packages/tinybird/src/client.ts
+++ b/packages/tinybird/src/client.ts
@@ -15,14 +15,14 @@ import {
export function publishPingResponse(tb: Tinybird) {
return tb.buildIngestEndpoint({
- datasource: "ping_response__v3",
+ datasource: "ping_response__v4",
event: tbIngestPingResponse,
});
}
export function getResponseList(tb: Tinybird) {
return tb.buildPipe({
- pipe: "response_list__v0",
+ pipe: "response_list__v1",
parameters: tbParameterResponseList,
data: tbBuildResponseList,
opts: {
@@ -33,7 +33,7 @@ export function getResponseList(tb: Tinybird) {
export function getMonitorList(tb: Tinybird) {
return tb.buildPipe({
- pipe: "monitor_list__v0",
+ pipe: "monitor_list__v1",
parameters: tbParameterMonitorList,
data: tbBuildMonitorList,
opts: {
@@ -49,7 +49,7 @@ export function getMonitorList(tb: Tinybird) {
*/
export function getHomeMonitorList(tb: Tinybird) {
return tb.buildPipe({
- pipe: "monitor_list__v0",
+ pipe: "monitor_list__v1",
parameters: tbParameterMonitorList,
data: tbBuildMonitorList,
opts: {
diff --git a/packages/tinybird/src/validation.ts b/packages/tinybird/src/validation.ts
index f40094ca4..32a9598cd 100644
--- a/packages/tinybird/src/validation.ts
+++ b/packages/tinybird/src/validation.ts
@@ -25,45 +25,37 @@ export const availableRegions = [
] as const;
/**
- * Values for the datasource ping_response__v3
+ * Values for the datasource ping_response__v4
*/
export const tbIngestPingResponse = z.object({
id: z.string(),
workspaceId: z.string(),
- pageId: z.string(),
monitorId: z.string(),
timestamp: z.number().int(),
statusCode: z.number().int(),
latency: z.number().int(), // in ms
cronTimestamp: z.number().int().optional().nullable().default(Date.now()),
url: z.string().url(),
- metadata: z.string().optional().default("{}").nullable(),
- region: z.string().min(4).max(4),
+ region: z.string().min(4).max(4), // REMINDER: won't work on fy
});
/**
- * Values from the pip response_list__v0
+ * Values from the pipe response_list__v1
*/
export const tbBuildResponseList = z.object({
id: z.string(),
workspaceId: z.string(),
- pageId: z.string(),
monitorId: z.string(),
timestamp: z.number().int(),
statusCode: z.number().int(),
latency: z.number().int(), // in ms
cronTimestamp: z.number().int().nullable().default(Date.now()),
url: z.string().url(),
- metadata: z
- .string()
- .default("{}")
- .transform((t) => JSON.parse(t))
- .nullable(),
region: z.enum(availableRegions),
});
/**
- * Params for pipe response_list__v0
+ * Params for pipe response_list__v1
*/
export const tbParameterResponseList = z.object({
monitorId: z.string().default(""), // REMINDER: remove default once alpha
@@ -75,24 +67,15 @@ export const tbParameterResponseList = z.object({
});
/**
- * All `groupBy` options for the monitoring list
- * - "day" will aggregate data within the same day
- * - "cron" will get data from single cron job
- */
-export const groupByRange = ["day", "cron"] as const;
-
-/**
- * Params for pipe monitor_list__v0
+ * Params for pipe monitor_list__v1
*/
export const tbParameterMonitorList = z.object({
- monitorId: z.string().optional().default(""), // REMINDER: remove default once alpha
- limit: z.number().int().optional().default(2500), // one day has 2448 pings (17 (regions) * 6 (per hour) * 24)
- cronTimestamp: z.number().int().optional(),
- groupBy: z.enum(groupByRange).optional(), // TODO: rename to frequency: z.enum(["1d", "auto"]) - where "auto" the default periodicity setup
+ monitorId: z.string(),
+ limit: z.number().int().default(60).optional(), // 40 days
});
/**
- * Values from the pipe monitor_list__v0
+ * Values from the pipe monitor_list__v1
*/
export const tbBuildMonitorList = z.object({
count: z.number().int(),
@@ -123,4 +106,3 @@ export type HomeStats = z.infer<typeof tbBuildHomeStats>;
export type ResponseListParams = z.infer<typeof tbParameterResponseList>;
export type MonitorListParams = z.infer<typeof tbParameterMonitorList>;
export type HomeStatsParams = z.infer<typeof tbParameterHomeStats>;
-export type GroupByRange = (typeof groupByRange)[number];