Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
feat: tb update schema (#472)
Maximilian Kaske4d213e8ff99b
diff --git a/apps/server/src/checker/checker.ts b/apps/server/src/checker/checker.tsindex 7d60736b7..fb992a1bd 100644--- a/apps/server/src/checker/checker.ts+++ b/apps/server/src/checker/checker.ts@@ -10,6 +10,7 @@ export const publishPingRetryPolicy = async ({payload,latency,statusCode,+ message,}: PublishPingType) => {try {console.log(@@ -17,14 +18,25 @@ export const publishPingRetryPolicy = async ({payload,)} with latency ${latency} and status code ${statusCode}`,);- await publishPing({ payload, statusCode, latency });+ await publishPing({+ payload,+ statusCode,+ latency,+ message,+ });} catch {try {console.log("2️⃣ try publish ping to tb - attempt 2 ",JSON.stringify(payload),);- await publishPing({ payload, statusCode, latency });++ await publishPing({+ payload,+ statusCode,+ latency,+ message,+ });} catch (e) {throw e;}@@ -40,6 +52,7 @@ const run = async (data: Payload, retry: number) => {let startTime = 0;let endTime = 0;let res = null;+ let message = undefined;// We are doing these for wrong urlstry {startTime = Date.now();@@ -47,6 +60,7 @@ const run = async (data: Payload, retry: number) => {endTime = Date.now();} catch (e) {endTime = Date.now();+ message = `${e}`;console.log(`🚨 error on pingEndpoint for ${JSON.stringify(data)} error: `,e,@@ -59,6 +73,7 @@ const run = async (data: Payload, retry: number) => {payload: data,latency,statusCode: res.status,+ message: undefined,});if (data?.status === "error") {await updateMonitorStatus({@@ -80,8 +95,10 @@ const run = async (data: Payload, retry: number) => {await publishPingRetryPolicy({payload: data,latency,- statusCode: res?.status || 0,+ statusCode: res?.status,+ message: message,});+if (data?.status === "active") {await updateMonitorStatus({monitorId: data.monitorId,diff --git a/apps/server/src/checker/ping.ts b/apps/server/src/checker/ping.tsindex 6bfe5e318..84559ee22 100644--- a/apps/server/src/checker/ping.ts+++ b/apps/server/src/checker/ping.ts@@ -1,5 +1,3 @@-import { nanoid } from "nanoid";-import { publishPingResponse } from "@openstatus/tinybird";import { env } from "../env";@@ -40,14 +38,16 @@ export async function pingEndpoint(data: Payload) {export type PublishPingType = {payload: Payload;- statusCode: number;latency: number;+ statusCode?: number | undefined;+ message?: string | undefined;};export async function publishPing({payload,- statusCode,latency,+ message,+ statusCode,}: PublishPingType) {const { monitorId, cronTimestamp, url, workspaceId } = payload;@@ -56,7 +56,6 @@ export async function publishPing({process.env.NODE_ENV === "test") {const res = await publishPingResponse({- id: nanoid(), // TBD: we don't need ittimestamp: Date.now(),statusCode,latency,@@ -65,6 +64,7 @@ export async function publishPing({monitorId,cronTimestamp,workspaceId,+ message: message,});if (res.successful_rows === 0) {throw new Error(`error 0 rows on publish ping for ${payload.monitorId}`);diff --git a/apps/server/src/checker/utils.ts b/apps/server/src/checker/utils.tsindex 6d7e4b804..c4454c12c 100644--- a/apps/server/src/checker/utils.ts+++ b/apps/server/src/checker/utils.ts@@ -18,7 +18,8 @@ type SendNotification = ({monitor: Monitor;notification: Notification;region: keyof typeof flyRegionsDict;- statusCode: number;+ statusCode?: number;+ message?: string;}) => Promise<void>;export const providerToFunction = {diff --git a/apps/web/src/app/_components/input-search.tsx b/apps/web/src/app/_components/input-search.tsxindex 3ae0512c4..cc95269cb 100644--- a/apps/web/src/app/_components/input-search.tsx+++ b/apps/web/src/app/_components/input-search.tsx@@ -59,7 +59,7 @@ export function InputSearch({},// defaultState{ limit: [10, 25, 50], status: [], region: [] } as {- status: number[];+ status: (number | null)[];limit: number[];region: string[];},diff --git a/apps/web/src/app/api/checker/regions/_checker.ts b/apps/web/src/app/api/checker/regions/_checker.tsindex 4b08a8847..6069f1077 100644--- a/apps/web/src/app/api/checker/regions/_checker.ts+++ b/apps/web/src/app/api/checker/regions/_checker.ts@@ -33,7 +33,6 @@ const monitor = async (const { monitorId, cronTimestamp, url, workspaceId } = payload;await publishPingResponse({- id: nanoid(), // TBD: we don't need ittimestamp: Date.now(),statusCode,monitorId,diff --git a/apps/web/src/app/monitor/[id]/page.tsx b/apps/web/src/app/monitor/[id]/page.tsxindex 9946f1535..8a23e6c34 100644--- a/apps/web/src/app/monitor/[id]/page.tsx+++ b/apps/web/src/app/monitor/[id]/page.tsx@@ -1,6 +1,6 @@import * as z from "zod";-import { availableRegions } from "@openstatus/tinybird";+import { availableRegions } from "@openstatus/utils";import { columns } from "@/components/data-table/columns";import { DataTable } from "@/components/data-table/data-table";diff --git a/apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsx b/apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsxindex 6866c2fbd..04a7b2f43 100644--- a/apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsx+++ b/apps/web/src/app/play/@modal/(..)monitor/[id]/page.tsx@@ -1,6 +1,6 @@import * as z from "zod";-import { availableRegions } from "@openstatus/tinybird";+import { availableRegions } from "@openstatus/utils";import { columns } from "@/components/data-table/columns";import { DataTable } from "@/components/data-table/data-table";diff --git a/apps/web/src/components/marketing/cards.tsx b/apps/web/src/components/marketing/cards.tsxindex dc8277051..891147177 100644--- a/apps/web/src/components/marketing/cards.tsx+++ b/apps/web/src/components/marketing/cards.tsx@@ -20,12 +20,12 @@ export function Cards(props: Feature) {return (<li key={i}><p className="flex flex-col">- <p>+ <span><FeatureIcon className="text-foreground/80 mb-1 mr-1.5 inline-flex h-4 w-4" /><span className="text-foreground font-medium">{feature.catchline.replace(".", "")}</span>{" "}- </p>+ </span><span className="text-muted-foreground">{feature.description}</span>diff --git a/apps/web/src/components/status-page/status-check.tsx b/apps/web/src/components/status-page/status-check.tsxindex 1fc6ee3ae..e558138e5 100644--- a/apps/web/src/components/status-page/status-check.tsx+++ b/apps/web/src/components/status-page/status-check.tsx@@ -52,6 +52,7 @@ export async function StatusCheck({function calcStatus() {const { count, ok } = monitorsData.flat(1).reduce((prev, curr) => {+ if (!curr.statusCode) return prev; // TODO: handle this betterconst isOk = curr.statusCode <= 299 && curr.statusCode >= 200;return { count: prev.count + 1, ok: prev.ok + (isOk ? 1 : 0) };},diff --git a/apps/web/src/content/changelog/sms-notification.mdx b/apps/web/src/content/changelog/sms-notification.mdxindex 368814166..3effa6109 100644--- a/apps/web/src/content/changelog/sms-notification.mdx+++ b/apps/web/src/content/changelog/sms-notification.mdx@@ -1,7 +1,7 @@---title: Get alerted on your cell phonedescription:- Receive alerts on your cell phone via SMS when your monitor is done.+ Receive alerts on your cell phone via SMS when your monitor is down.publishedAt: 2023-11-13image: /assets/changelog/sms-notification.png---diff --git a/packages/notifications/discord/src/index.ts b/packages/notifications/discord/src/index.tsindex 982d4ed13..f7a7d871c 100644--- a/packages/notifications/discord/src/index.ts+++ b/packages/notifications/discord/src/index.ts@@ -20,11 +20,13 @@ export const sendDiscordMessage = async ({notification,region,statusCode,+ message,}: {monitor: Monitor;notification: Notification;- statusCode: number;+ statusCode?: number;region: string;+ message?: string;}) => {const notificationData = JSON.parse(notification.data);const { discord: webhookUrl } = notificationData; // webhook url@@ -34,7 +36,9 @@ export const sendDiscordMessage = async ({await postToWebhook(`Your monitor ${name} is down 🚨- Your monitor with url ${monitor.url} is down in ${region} with status code ${statusCode}.`,+ Your monitor with url ${monitor.url} is down in ${region} with ${+ statusCode ? `status code ${statusCode}` : `error message ${message}`+ }.`,webhookUrl,);} catch (err) {diff --git a/packages/notifications/email/src/index.ts b/packages/notifications/email/src/index.tsindex 0a3898163..261bf0673 100644--- a/packages/notifications/email/src/index.ts+++ b/packages/notifications/email/src/index.ts@@ -8,11 +8,13 @@ export const send = async ({notification,region,statusCode,+ message,}: {monitor: Monitor;notification: Notification;- statusCode: number;+ statusCode?: number;region: string;+ message?: string;}) => {// FIXME:const config = EmailConfigurationSchema.parse(JSON.parse(notification.data));@@ -29,7 +31,13 @@ export const send = async ({from: "Notifications <[email protected]>",subject: `Your monitor ${monitor.name} is down 🚨`,- html: `<p>Hi,<br><br>Your monitor ${monitor.name} is down in ${region}. </p><p>URL : ${monitor.url}</p><p>Status Code: ${statusCode}</p><p>OpenStatus 🏓 </p>`,+ html: `<p>Hi,<br><br>Your monitor ${+ monitor.name+ } is down in ${region}. </p><p>URL : ${monitor.url}</p> ${+ statusCode+ ? `<p>Status Code: ${statusCode}</p>`+ : `<p>Error message: ${message}</p>`+ }<p>OpenStatus 🏓 </p>`,}),});diff --git a/packages/notifications/slack/src/index.ts b/packages/notifications/slack/src/index.tsindex 3e6dd8bbd..bbd7984fc 100644--- a/packages/notifications/slack/src/index.ts+++ b/packages/notifications/slack/src/index.ts@@ -17,11 +17,13 @@ export const sendSlackMessage = async ({notification,region,statusCode,+ message,}: {monitor: Monitor;notification: Notification;- statusCode: number;+ statusCode?: number;region: string;+ message?: string;}) => {const notificationData = JSON.parse(notification.data);const { slack: webhookUrl } = notificationData; // webhook url@@ -35,7 +37,13 @@ export const sendSlackMessage = async ({type: "section",text: {type: "mrkdwn",- text: `Your monitor <${monitor.url}/|${name}> is down in ${region} with status code ${statusCode} 🚨 \n\n Powered by <https://www.openstatus.dev/|OpenStatus>.`,+ text: `Your monitor <${+ monitor.url+ }/|${name}> is down in ${region} with ${+ statusCode+ ? `status code ${statusCode}`+ : `error message ${message}`+ } 🚨 \n\n Powered by <https://www.openstatus.dev/|OpenStatus>.`,},accessory: {type: "button",diff --git a/packages/notifications/twillio-sms/src/index.ts b/packages/notifications/twillio-sms/src/index.tsindex 173fed415..f06e6f355 100644--- a/packages/notifications/twillio-sms/src/index.ts+++ b/packages/notifications/twillio-sms/src/index.ts@@ -8,11 +8,13 @@ export const sendTextMessage = async ({notification,region,statusCode,+ message,}: {monitor: Monitor;notification: Notification;- statusCode: number;+ statusCode?: number;region: string;+ message?: string;}) => {const notificationData = SmsConfigurationSchema.parse(JSON.parse(notification.data),@@ -24,7 +26,9 @@ export const sendTextMessage = async ({body.set("From", "+14807252613");body.set("Body",- `Your monitor ${name} / ${monitor.url} is down in ${region} with status code ${statusCode}`,+ `Your monitor ${name} / ${monitor.url} is down in ${region} with ${+ statusCode ? `status code ${statusCode}` : `error: ${message}`+ }`,);try {diff --git a/packages/tinybird/README.md b/packages/tinybird/README.mdnew file mode 100644index 000000000..00a23b079--- /dev/null+++ b/packages/tinybird/README.md@@ -0,0 +1,35 @@+### A guide on how to migrate your tinybird datasource++> What to do when you want to add/remove/update a column in your `datasource`.++The `_migration` folder includes:++- `ping_response__v4.datasource` which represents the `VERSION 4` of our+ datasource+- `ping_response.datasource` which has the upgraded schema and a new `VERSION 5`+- `tb_backfill_populate.pipe` will fill the datasource with all the data until a+ given timestamp+- `tb_materialized_until_change_ingest.pipe` will fill the data from a given+ timestamp++```+tb push _migration/ping_response.datasource+tb push _migration/tb_materialized_until_change_ingest.pipe+# after the given ts, it is time to run the backfill populate+tb push _migration/tb_backfill_populate.pipe --populate --wait+# after populate ends, it is time to remove the pipe+tb pipe rm tb_backfill_populate --yes+```++Check if all the rows have been migrated:++```+tb pipe _migration/tb_datasource_union.pipe+# after checking the result of the pipe+tb pipe rm tb_datasource_union.pipe --yes+```++---++Link to the [issue](https://github.com/openstatusHQ/openstatus/issues/278) from+Gonzalo as reference.diff --git a/packages/tinybird/_migration/ping_response.datasource b/packages/tinybird/_migration/ping_response.datasourcenew file mode 100644index 000000000..52cbe6ab2--- /dev/null+++ b/packages/tinybird/_migration/ping_response.datasource@@ -0,0 +1,16 @@+VERSION 5++SCHEMA >+ `latency` Int64 `json:$.latency`,+ `monitorId` String `json:$.monitorId`,+ `region` LowCardinality(String) `json:$.region`,+ `statusCode` Nullable(Int16) `json:$.statusCode`,+ `timestamp` Int64 `json:$.timestamp`,+ `url` String `json:$.url`,+ `workspaceId` String `json:$.workspaceId`,+ `cronTimestamp` Int64 `json:$.cronTimestamp`,+ `message` Nullable(String) `json:$.message`++ENGINE "MergeTree"+ENGINE_SORTING_KEY "monitorId, cronTimestamp"+ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(cronTimestamp))"\ No newline at end of filediff --git a/packages/tinybird/_migration/ping_response__v4.datasource b/packages/tinybird/_migration/ping_response__v4.datasourcenew file mode 100644index 000000000..dc55dea56--- /dev/null+++ b/packages/tinybird/_migration/ping_response__v4.datasource@@ -0,0 +1,16 @@+VERSION 4++SCHEMA >+ `id` String `json:$.id`,+ `latency` Int16 `json:$.latency`,+ `monitorId` String `json:$.monitorId`,+ `region` LowCardinality(String) `json:$.region`,+ `statusCode` Int16 `json:$.statusCode`,+ `timestamp` Int64 `json:$.timestamp`,+ `url` String `json:$.url`,+ `workspaceId` String `json:$.workspaceId`,+ `cronTimestamp` Int64 `json:$.cronTimestamp`++ENGINE "MergeTree"+ENGINE_SORTING_KEY "monitorId, cronTimestamp"+ENGINE_PARTITION_KEY "toYYYYMM(fromUnixTimestamp64Milli(cronTimestamp))"\ No newline at end of filediff --git a/packages/tinybird/_migration/tb_backfill_populate.pipe b/packages/tinybird/_migration/tb_backfill_populate.pipenew file mode 100644index 000000000..ef71f5a2a--- /dev/null+++ b/packages/tinybird/_migration/tb_backfill_populate.pipe@@ -0,0 +1,17 @@+NODE mat_node+SQL >++ SELECT+ latency,+ monitorId,+ toLowCardinality(region) region,+ statusCode,+ timestamp,+ url,+ workspaceId,+ cronTimestamp,+ FROM ping_response__v4+ WHERE fromUnixTimestamp64Milli(cronTimestamp) <= '2023-11-13 18:16:00.000'++TYPE materialized+DATASOURCE ping_response__v5\ No newline at end of filediff --git a/packages/tinybird/_migration/tb_datasource_union.pipe b/packages/tinybird/_migration/tb_datasource_union.pipenew file mode 100644index 000000000..390b5daf7--- /dev/null+++ b/packages/tinybird/_migration/tb_datasource_union.pipe@@ -0,0 +1,6 @@+NODE union_all+SQL >++ SELECT count() FROM ping_response__v5+ UNION ALL+ SELECT count() FROM ping_response__v4diff --git a/packages/tinybird/_migration/tb_materialized_until_change_ingest.pipe b/packages/tinybird/_migration/tb_materialized_until_change_ingest.pipenew file mode 100644index 000000000..16142297a--- /dev/null+++ b/packages/tinybird/_migration/tb_materialized_until_change_ingest.pipe@@ -0,0 +1,17 @@+NODE mat_node+SQL >++ SELECT+ latency,+ monitorId,+ toLowCardinality(region) region,+ statusCode,+ timestamp,+ url,+ workspaceId,+ cronTimestamp,+ FROM ping_response__v4+ WHERE fromUnixTimestamp64Milli(cronTimestamp) > '2023-11-13 18:16:00.000'++TYPE materialized+DATASOURCE ping_response__v5\ No newline at end of filediff --git a/packages/tinybird/package.json b/packages/tinybird/package.jsonindex 9e96d7735..da5385634 100644--- a/packages/tinybird/package.json+++ b/packages/tinybird/package.json@@ -9,6 +9,7 @@},"devDependencies": {"@openstatus/tsconfig": "workspace:*",+ "@openstatus/utils": "workspace:*","@types/node": "20.8.0","typescript": "5.2.2"}diff --git a/packages/tinybird/pipes/home_stats.pipe b/packages/tinybird/pipes/home_stats.pipeindex 5955f688a..4a5573d85 100644--- a/packages/tinybird/pipes/home_stats.pipe+++ b/packages/tinybird/pipes/home_stats.pipe@@ -5,7 +5,7 @@ SQL >%SELECT COUNT(*) as count- FROM ping_response__v4+ FROM ping_response__v5{% if defined(period) %}{% if String(period) == "1h" %}WHERE cronTimestamp > toUnixTimestamp(now() - INTERVAL 1 HOUR) * 1000diff --git a/packages/tinybird/pipes/response_list.pipe b/packages/tinybird/pipes/response_list.pipeindex 593c2c35f..d2fc2b126 100644--- a/packages/tinybird/pipes/response_list.pipe+++ b/packages/tinybird/pipes/response_list.pipe@@ -1,11 +1,11 @@-VERSION 1+VERSION 2NODE response_list_0SQL >%- SELECT id, latency, monitorId, region, statusCode, timestamp, url, workspaceId, cronTimestamp- FROM ping_response+ SELECT latency, monitorId, region, statusCode, timestamp, url, workspaceId, cronTimestamp+ FROM ping_response__v5WHERE monitorId = {{ String(monitorId, 'openstatusPing') }}{% if defined(region) %}AND region = {{ String(region) }}diff --git a/packages/tinybird/pipes/status_timezone.pipe b/packages/tinybird/pipes/status_timezone.pipeindex 820f06b85..bc86bd481 100644--- a/packages/tinybird/pipes/status_timezone.pipe+++ b/packages/tinybird/pipes/status_timezone.pipe@@ -16,7 +16,7 @@ SQL >avg(latency) AS avgLatency,count() AS count,count(multiIf((statusCode >= 200) AND (statusCode <= 299), 1, NULL)) AS ok- FROM ping_response__v4+ FROM ping_response__v5WHERE(day IS NOT NULL)AND (day != 0)diff --git a/packages/tinybird/src/client.ts b/packages/tinybird/src/client.tsindex 2b5109f51..7dc0adb52 100644--- a/packages/tinybird/src/client.ts+++ b/packages/tinybird/src/client.ts@@ -14,13 +14,13 @@ import {const tb = new Tinybird({ token: process.env.TINY_BIRD_API_KEY! });export const publishPingResponse = tb.buildIngestEndpoint({- datasource: "ping_response__v4",+ datasource: "ping_response__v5",event: tbIngestPingResponse,});export function getResponseList(tb: Tinybird) {return tb.buildPipe({- pipe: "response_list__v1",+ pipe: "response_list__v2",parameters: tbParameterResponseList,data: tbBuildResponseList,opts: {diff --git a/packages/tinybird/src/validation.ts b/packages/tinybird/src/validation.tsindex e7d1766ee..2c5e98264 100644--- a/packages/tinybird/src/validation.ts+++ b/packages/tinybird/src/validation.ts@@ -1,62 +1,39 @@import * as z from "zod";-export const vercelRegions = [- "arn1",- "bom1",- "cdg1",- "cle1",- "cpt1",- "dub1",- "fra1",- "gru1",- "hkg1",- "hnd1",- "iad1",- "icn1",- "kix1",- "lhr1",- "pdx1",- "sfo1",- "sin1",- "syd1",-] as const;--export const flyRegions = ["ams", "iad", "hkg", "jnb", "syd", "gru"] as const;--export const availableRegions = [...vercelRegions, ...flyRegions] as const;+import { availableRegions } from "@openstatus/utils";/**- * Values for the datasource ping_response__v4+ * Values for the datasource ping_response*/export const tbIngestPingResponse = z.object({- id: z.string(),workspaceId: z.string(),monitorId: z.string(),timestamp: z.number().int(),- statusCode: z.number().int(),+ statusCode: z.number().int().nullable().optional(),latency: z.number(), // in mscronTimestamp: z.number().int().optional().nullable().default(Date.now()),url: z.string().url(),region: z.string().min(3).max(4), // REMINDER: won't work on fy+ message: z.string().nullable().optional(),});/**- * Values from the pipe response_list__v1+ * Values from the pipe response_list*/export const tbBuildResponseList = z.object({- id: z.string(),workspaceId: z.string(),monitorId: z.string(),timestamp: z.number().int(),- statusCode: z.number().int(),+ statusCode: z.number().int().nullable().default(null),latency: z.number().int(), // in mscronTimestamp: z.number().int().nullable().default(Date.now()),url: z.string().url(),region: z.enum(availableRegions),+ message: z.string().nullable().optional(),});/**- * Params for pipe response_list__v1+ * Params for pipe response_list*/export const tbParameterResponseList = z.object({monitorId: z.string().default(""), // REMINDER: remove default once alpha@@ -68,7 +45,7 @@ export const tbParameterResponseList = z.object({});/**- * Params for pipe status_timezone__v0+ * Params for pipe status_timezone*/export const tbParameterMonitorList = z.object({monitorId: z.string(),@@ -77,7 +54,7 @@ export const tbParameterMonitorList = z.object({});/**- * Values from the pipe status_timezone__v0+ * Values from the pipe status_timezone*/export const tbBuildMonitorList = z.object({count: z.number().int(),@@ -87,7 +64,7 @@ export const tbBuildMonitorList = z.object({});/**- * Params for pipe home_stats__v0+ * Params for pipe home_stats*/export const tbParameterHomeStats = z.object({cronTimestamp: z.number().int().optional(),@@ -95,7 +72,7 @@ export const tbParameterHomeStats = z.object({});/**- * Values from the pipe home_stats__v0+ * Values from the pipe home_stats*/export const tbBuildHomeStats = z.object({count: z.number().int(),diff --git a/packages/utils/index.ts b/packages/utils/index.tsindex 71ef19586..5c12d4694 100644--- a/packages/utils/index.ts+++ b/packages/utils/index.ts@@ -160,4 +160,6 @@ export const vercelRegions = [export const flyRegions = ["ams", "iad", "hkg", "jnb", "syd", "gru"] as const;+export const availableRegions = [...vercelRegions, ...flyRegions] as const;+export const regionsDict = { ...vercelRegionsDict, ...flyRegionsDict } as const;diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 049a1c313..7db3890ec 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -754,6 +754,9 @@ importers:'@openstatus/tsconfig':specifier: workspace:*version: link:../tsconfig+ '@openstatus/utils':+ specifier: workspace:*+ version: link:../utils'@types/node':specifier: 20.8.0version: 20.8.0