Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
π₯ Go tweak (#495)
Thibault Le Ouay920ab4f4ecf1
diff --git a/apps/checker/README.md b/apps/checker/README.mdnew file mode 100644index 000000000..61674c22c--- /dev/null+++ b/apps/checker/README.md@@ -0,0 +1,30 @@+# OpenStatus Checker++The checker service to ping external service.++It pings the service and save thedata to the tinybird++## How to run++```bash+go run *.go+```++## How to build++```bash+go build -o checker *.go+```++## How to run in docker++```bash+docker build -t checker .+docker run -p 8080:8080 checker+```++## How to deploy++```bash+fly deploy+```diff --git a/apps/checker/fly.toml b/apps/checker/fly.tomlindex 0de4169dd..5d89197fe 100644--- a/apps/checker/fly.toml+++ b/apps/checker/fly.toml@@ -19,12 +19,19 @@ primary_region = "ams"[http_service]internal_port = 8080force_https = true- auto_stop_machines = true- auto_start_machines = true- min_machines_running = 0+ auto_stop_machines = false+ auto_start_machines = falseprocesses = ["app"][[vm]]cpu_kind = "shared"- cpus = 2- memory_mb = 512+ cpus = 1+ memory_mb = 256+++[[http_service.checks]]+ grace_period = "10s"+ interval = "15s"+ method = "GET"+ timeout = "5s"+ path = "/ping"\ No newline at end of filediff --git a/apps/checker/main.go b/apps/checker/main.goindex 86f74c20a..5be3fbbc8 100644--- a/apps/checker/main.go+++ b/apps/checker/main.go@@ -1,12 +1,11 @@package mainimport (- "bytes""encoding/json""fmt""net/http""os"- "time"+ "strconv""github.com/go-chi/chi/v5""github.com/go-chi/chi/v5/middleware"@@ -31,79 +30,165 @@ func main() {r := chi.NewRouter()r.Use(middleware.Logger)r.Post("/", func(w http.ResponseWriter, r *http.Request) {- if r.Header.Get("Authorization") != "Basic "+ os.Getenv("CRON_SECRET") {+ if r.Header.Get("Authorization") != "Basic "+os.Getenv("CRON_SECRET") {http.Error(w, "Unauthorized", 401)return- }- region := os.Getenv("FLY_REGION")+ }+ i, err := strconv.Atoi(r.Header.Get("X-CloudTasks-TaskRetryCount"))+ if err != nil {+ http.Error(w, "Something went whont", 400)+ return+ }+ // If something went wrong we only try it twice+ if i > 1 {+ w.WriteHeader(http.StatusOK)+ w.Write([]byte("Ok"))+ return+ }+if r.Body == nil {http.Error(w, "Please send a request body", 400)return}var u InputData+ // region := os.Getenv("FLY_REGION")- err := json.NewDecoder(r.Body).Decode(&u)+ err = json.NewDecoder(r.Body).Decode(&u)- fmt.Printf("Start checker for %+v", u)+ fmt.Printf("π Start checker for %+v \n", u)if err != nil {- http.Error(w, err.Error(), 400)+ w.Write([]byte("Ok"))+ w.WriteHeader(200)return}- request, error := http.NewRequest(u.Method, u.Url, bytes.NewReader([]byte(u.Body)))- // Setting headers- for _, header := range u.Headers {- fmt.Printf("%+v", header)- if header.Key != "" && header.Value != "" {- request.Header.Set(header.Key, header.Value)+ client := &http.Client{}+ defer client.CloseIdleConnections()++ response, error := ping(client, u)++ if error != nil {+ sendToTinybird(response)+ if u.Status == "active" {+ // updateStatus(UpdateData{+ // MonitorId: u.MonitorId,+ // Status: "error",+ // Message: error.Error(),+ // Region: region,+ // })}+ w.Write([]byte("Ok"))+ w.WriteHeader(200)+ return}- if error != nil {- fmt.Println(error)+ sendToTinybird(response)++ if response.StatusCode < 200 || response.StatusCode >= 300 {+ // If the status code is not within the 200 range, we update the status to error+ // updateStatus(UpdateData{+ // MonitorId: u.MonitorId,+ // Status: "error",+ // StatusCode: response.StatusCode,+ // Region: region,+ // })+ }+ if u.Status == "error" {+ // If the status was error, we update it to active+ // updateStatus(UpdateData{+ // MonitorId: u.MonitorId,+ // Status: "active",+ // Region: region,+ // StatusCode: response.StatusCode,+ // })+ }++ fmt.Printf("β±οΈ End checker for %+v with latency %+d and statusCode %+d", u, response.Latency, response.StatusCode)+ w.Write([]byte("Ok"))+ w.WriteHeader(200)+ return+ })+ // That's the new checker sending to the correct ingest endpoint+ r.Post("/checker", func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Authorization") != "Basic "+os.Getenv("CRON_SECRET") {+ http.Error(w, "Unauthorized", 401)+ return+ }+ i, err := strconv.Atoi(r.Header.Get("X-CloudTasks-TaskRetryCount"))+ if err != nil {+ http.Error(w, "Something went whont", 400)+ return+ }+ // If something went wrong we only try it twice+ if i > 1 {+ w.WriteHeader(http.StatusOK)+ w.Write([]byte("Ok"))+ return+ }++ if r.Body == nil {+ http.Error(w, "Please send a request body", 400)+ return+ }+ var u InputData+ // region := os.Getenv("FLY_REGION")++ err = json.NewDecoder(r.Body).Decode(&u)++ fmt.Printf("π Start checker for %+v \n", u)++ if err != nil {+ w.Write([]byte("Ok"))+ w.WriteHeader(200)+ return}client := &http.Client{}- start := time.Now().UTC().UnixMilli()- response, error := client.Do(request)- end := time.Now().UTC().UnixMilli()+ defer client.CloseIdleConnections()++ response, error := ping(client, u)- // Retry if errorif error != nil {- response, error = client.Do(request)- end = time.Now().UTC().UnixMilli()+ sendToTinybirdNew(response)+ if u.Status == "active" {+ // updateStatus(UpdateData{+ // MonitorId: u.MonitorId,+ // Status: "error",+ // Message: error.Error(),+ // Region: region,+ // })+ }+ w.Write([]byte("Ok"))+ w.WriteHeader(200)+ return}- latency := end - start- fmt.Println("π Checked url: %v with latency %v in region %v ", u.Url, latency, region)- fmt.Printf("Response %+v for %+v", response, u)- if error != nil {- tiny((PingData{- Latency: (latency),- MonitorId: u.MonitorId,- Region: region,- WorkspaceId: u.WorkspaceId,- Timestamp: time.Now().UTC().UnixMilli(),- Url: u.Url,- Message: error.Error(),- }))- } else {- tiny((PingData{- Latency: (latency),- MonitorId: u.MonitorId,- Region: region,- WorkspaceId: u.WorkspaceId,- StatusCode: int16(response.StatusCode),- Timestamp: time.Now().UTC().UnixMilli(),- Url: u.Url,- }))- }-- fmt.Printf("End checker for %+v", u)+ sendToTinybirdNew(response)++ if response.StatusCode < 200 || response.StatusCode >= 300 {+ // If the status code is not within the 200 range, we update the status to error+ // updateStatus(UpdateData{+ // MonitorId: u.MonitorId,+ // Status: "error",+ // StatusCode: response.StatusCode,+ // Region: region,+ // })+ }+ if u.Status == "error" {+ // If the status was error, we update it to active+ // updateStatus(UpdateData{+ // MonitorId: u.MonitorId,+ // Status: "active",+ // Region: region,+ // StatusCode: response.StatusCode,+ // })+ }+ fmt.Printf("β±οΈ End checker for %+v with latency %+d and statusCode %+d", u, response.Latency, response.StatusCode)w.Write([]byte("Ok"))w.WriteHeader(200)+ return})r.Get("/ping", func(w http.ResponseWriter, r *http.Request) {@@ -116,8 +201,9 @@ func main() {}w.Header().Set("Content-Type", "application/json")- w.WriteHeader(http.StatusCreated)+ w.WriteHeader(http.StatusOK)json.NewEncoder(w).Encode(data)+ return})http.ListenAndServe(":8080", r)diff --git a/apps/checker/ping.go b/apps/checker/ping.goindex 0b4902dc9..8c55c7769 100644--- a/apps/checker/ping.go+++ b/apps/checker/ping.go@@ -6,6 +6,7 @@ import ("fmt""io""net/http"+ "net/url""os""time")@@ -14,17 +15,17 @@ type PingData struct {WorkspaceId string `json:"workspaceId"`MonitorId string `json:"monitorId"`Timestamp int64 `json:"timestamp"`- StatusCode int16 `json:"statusCode"`+ StatusCode int `json:"statusCode,omitempty"`Latency int64 `json:"latency"`CronTimestamp int64 `json:"cronTimestamp"`Url string `json:"url"`Region string `json:"region"`- Message string `json:"message"`+ Message string `json:"message,omitempty"`}-func tiny(pingData PingData) {- url := "https://api.tinybird.co/v0/events?name=golang_ping_response__v1"- fmt.Println("URL:>", url)+func sendToTinybird(pingData PingData) {+ url := "https://api.tinybird.co/v0/events?name=golang_ping_response__v3"+ fmt.Printf("π Sending data to Tinybird for %+v \n", pingData)bearer := "Bearer " + os.Getenv("TINYBIRD_TOKEN")payloadBuf := new(bytes.Buffer)json.NewEncoder(payloadBuf).Encode(pingData)@@ -33,13 +34,84 @@ func tiny(pingData PingData) {req.Header.Set("Content-Type", "application/json")client := &http.Client{Timeout: time.Second * 10}- resp, err := client.Do(req)+ _, err = client.Do(req)if err != nil {fmt.Println(err)+ panic(err)+ }+ // Should we add a retry mechanism here?++}++func sendToTinybirdNew(pingData PingData) {+ url := "https://api.tinybird.co/v0/events?name=ping_response__v5"+ fmt.Printf("π Sending data to Tinybird for %+v \n", pingData)+ bearer := "Bearer " + os.Getenv("TINYBIRD_TOKEN")+ payloadBuf := new(bytes.Buffer)+ json.NewEncoder(payloadBuf).Encode(pingData)+ req, err := http.NewRequest("POST", url, payloadBuf)+ req.Header.Set("Authorization", bearer)+ req.Header.Set("Content-Type", "application/json")++ client := &http.Client{Timeout: time.Second * 10}+ _, err = client.Do(req)+ if err != nil {+ fmt.Println(err)+ panic(err)+ }+ // Should we add a retry mechanism here?++}++func ping(client *http.Client, inputData InputData) (PingData, error) {++ region := os.Getenv("FLY_REGION")+ request, err := http.NewRequest(inputData.Method, inputData.Url, bytes.NewReader([]byte(inputData.Body)))++ if err != nil {+ return PingData{}, fmt.Errorf("Unable to create request: %w", err)+ }+ request.Header.Set("User-Agent", "OpenStatus/1.0")++ // Setting headers+ for _, header := range inputData.Headers {+ if header.Key != "" && header.Value != "" {+ request.Header.Set(header.Key, header.Value)+ }+ }++ start := time.Now()+ response, err := client.Do(request)+ latency := time.Since(start).Milliseconds()+ defer response.Body.Close()++ _, err = io.ReadAll(response.Body)+ if err != nil {+ if urlErr, ok := err.(*url.Error); ok {+ if urlErr.Timeout() {+ return PingData{+ Latency: latency,+ MonitorId: inputData.MonitorId,+ Region: region,+ WorkspaceId: inputData.WorkspaceId,+ Timestamp: time.Now().UTC().UnixMilli(),+ Url: inputData.Url,+ Message: fmt.Sprintf("Timeout after %d ms", latency),+ }, nil+ }+ }+ return PingData{}, fmt.Errorf("Error while reading body from %s: %w", inputData.Url, err)}- defer resp.Body.Close()- body, _ := io.ReadAll(resp.Body)- fmt.Println(string(body))+ return PingData{+ Latency: latency,+ StatusCode: response.StatusCode,+ MonitorId: inputData.MonitorId,+ Region: region,+ WorkspaceId: inputData.WorkspaceId,+ Timestamp: time.Now().UTC().UnixMilli(),+ CronTimestamp: inputData.CronTimestamp,+ Url: inputData.Url,+ }, nil}diff --git a/apps/checker/update.go b/apps/checker/update.gonew file mode 100644index 000000000..77696e2d3--- /dev/null+++ b/apps/checker/update.go@@ -0,0 +1,36 @@+package main++import (+ "bytes"+ "encoding/json"+ "fmt"+ "net/http"+ "os"+ "time"+)++type UpdateData struct {+ MonitorId string `json:"monitorId"`+ Status string `json:"status"`+ Message string `json:"message,omitempty"`+ StatusCode int `json:"statusCode,omitempty"`+ Region string `json:"region"`+}++func updateStatus(updateData UpdateData) {+ url := "https://openstatus-api.fly.dev/updateStatus"+ fmt.Println("URL:>", url)+ bearer := "Bearer " + os.Getenv("CRON_SECRET")+ payloadBuf := new(bytes.Buffer)+ json.NewEncoder(payloadBuf).Encode(updateData)+ req, err := http.NewRequest("POST", url, payloadBuf)+ req.Header.Set("Authorization", bearer)+ req.Header.Set("Content-Type", "application/json")++ client := &http.Client{Timeout: time.Second * 10}+ _, err = client.Do(req)+ if err != nil {+ panic(err)+ }+ // Should we add a retry mechanism here?+}diff --git a/apps/server/src/checker/alerting.ts b/apps/server/src/checker/alerting.tsindex b103465f5..05abc42c3 100644--- a/apps/server/src/checker/alerting.ts+++ b/apps/server/src/checker/alerting.ts@@ -1,12 +1,14 @@import { db, eq, schema } from "@openstatus/db";-import type { MonitorRegion, MonitorStatus } from "@openstatus/db/src/schema";+import type {+ MonitorFlyRegion,+ MonitorStatus,+} from "@openstatus/db/src/schema";import {selectMonitorSchema,selectNotificationSchema,} from "@openstatus/db/src/schema";import { flyRegionsDict } from "@openstatus/utils";-import { env } from "../env";import { checkerAudit } from "../utils/audit-log";import { providerToFunction } from "./utils";@@ -58,11 +60,12 @@ export const triggerAlerting = async ({export const upsertMonitorStatus = async ({monitorId,status,+ region,}: {monitorId: string;status: MonitorStatus;+ region: MonitorFlyRegion;}) => {- const region = env.FLY_REGION as MonitorRegion;await db.insert(schema.monitorStatusTable).values({ status, region, monitorId: Number(monitorId) })diff --git a/apps/server/src/checker/index.ts b/apps/server/src/checker/index.tsindex ee907c520..7f821d1c2 100644--- a/apps/server/src/checker/index.ts+++ b/apps/server/src/checker/index.ts@@ -1,6 +1,11 @@import { Hono } from "hono";+import { z } from "zod";++import { flyRegions } from "@openstatus/db/src/schema/monitors/constants";import { env } from "../env";+import { checkerAudit } from "../utils/audit-log";+import { upsertMonitorStatus } from "./alerting";import { checkerRetryPolicy } from "./checker";import { payloadSchema } from "./schema";import type { Payload } from "./schema";@@ -113,3 +118,67 @@ checkerRoute.post("/checkerV2", async (c) => {return c.text("Internal Server Error", 500);}});++checkerRoute.post("/statusChange", async (c) => {+ const auth = c.req.header("Authorization");+ if (auth !== `Basic ${env.CRON_SECRET}`) {+ console.error("Unauthorized");+ return c.text("Unauthorized", 401);+ }++ const json = await c.req.json();+ const schema = z.object({+ monitorId: z.string(),+ status: z.enum(["active", "error"]), // that's the new status+ message: z.string().optional(),+ statusCode: z.number().optional(),+ region: z.enum(flyRegions),+ });++ const result = schema.safeParse(json);+ if (!result.success) {+ // console.error(result.error);+ return c.text("Unprocessable Entity", 422);+ }+ const { monitorId, status, message, region, statusCode } = result.data;++ console.log(`π update monitor status ${JSON.stringify(result.data)}`);++ switch (status) {+ case "active":+ await upsertMonitorStatus({+ monitorId: monitorId,+ status: "active",+ region: region,+ });+ if (!statusCode) {+ return;+ }+ await checkerAudit.publishAuditLog({+ id: `monitor:${monitorId}`,+ action: "monitor.recovered",+ targets: [{ id: monitorId, type: "monitor" }],+ metadata: { region: region, statusCode: statusCode },+ });+ break;+ case "error":+ await upsertMonitorStatus({+ monitorId: monitorId,+ status: "error",+ region: region,+ });+ // ALPHA+ await checkerAudit.publishAuditLog({+ id: `monitor:${monitorId}`,+ action: "monitor.failed",+ targets: [{ id: monitorId, type: "monitor" }],+ metadata: {+ region: region,+ statusCode: statusCode,+ message,+ },+ });+ break;+ }+ return c.text("Ok", 200);+});diff --git a/packages/db/src/schema/monitors/validation.ts b/packages/db/src/schema/monitors/validation.tsindex 127242e28..954bffbea 100644--- a/packages/db/src/schema/monitors/validation.ts+++ b/packages/db/src/schema/monitors/validation.ts@@ -2,6 +2,7 @@ import { createInsertSchema, createSelectSchema } from "drizzle-zod";import { z } from "zod";import {+ flyRegions,monitorJobTypes,monitorMethods,monitorPeriodicity,@@ -15,6 +16,7 @@ export const monitorMethodsSchema = z.enum(monitorMethods);export const monitorStatusSchema = z.enum(monitorStatus);export const monitorRegionSchema = z.enum(monitorRegions);export const monitorJobTypesSchema = z.enum(monitorJobTypes);+export const monitorFlyRegionSchema = z.enum(flyRegions);// TODO: shared functionfunction stringToArrayProcess<T>(string: T) {}@@ -79,4 +81,5 @@ export type MonitorStatus = z.infer<typeof monitorStatusSchema>;export type MonitorPeriodicity = z.infer<typeof monitorPeriodicitySchema>;export type MonitorMethod = z.infer<typeof monitorMethodsSchema>;export type MonitorRegion = z.infer<typeof monitorRegionSchema>;+export type MonitorFlyRegion = z.infer<typeof monitorFlyRegionSchema>;export type MonitorJobType = z.infer<typeof monitorJobTypesSchema>;