Skip to content

Open-source status page, incident response and uptime monitoring

πŸ”₯ Go tweak (#495)

Thibault Le Ouay920ab4f4ecf1
diff --git a/apps/checker/README.md b/apps/checker/README.md
new file mode 100644
index 000000000..61674c22c
--- /dev/null
+++ b/apps/checker/README.md
@@ -0,0 +1,30 @@
+# OpenStatus Checker
+
+The checker service to ping external service.
+
+It pings the service and save thedata to the tinybird
+
+## How to run
+
+```bash
+go run *.go
+```
+
+## How to build
+
+```bash
+go build -o checker *.go
+```
+
+## How to run in docker
+
+```bash
+docker build -t checker .
+docker run -p 8080:8080 checker
+```
+
+## How to deploy
+
+```bash
+fly deploy
+```
diff --git a/apps/checker/fly.toml b/apps/checker/fly.toml
index 0de4169dd..5d89197fe 100644
--- a/apps/checker/fly.toml
+++ b/apps/checker/fly.toml
@@ -19,12 +19,19 @@ primary_region = "ams"
[http_service]
internal_port = 8080
force_https = true
- auto_stop_machines = true
- auto_start_machines = true
- min_machines_running = 0
+ auto_stop_machines = false
+ auto_start_machines = false
processes = ["app"]
[[vm]]
cpu_kind = "shared"
- cpus = 2
- memory_mb = 512
+ cpus = 1
+ memory_mb = 256
+
+
+[[http_service.checks]]
+ grace_period = "10s"
+ interval = "15s"
+ method = "GET"
+ timeout = "5s"
+ path = "/ping"
\ No newline at end of file
diff --git a/apps/checker/main.go b/apps/checker/main.go
index 86f74c20a..5be3fbbc8 100644
--- a/apps/checker/main.go
+++ b/apps/checker/main.go
@@ -1,12 +1,11 @@
package main
import (
- "bytes"
"encoding/json"
"fmt"
"net/http"
"os"
- "time"
+ "strconv"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
@@ -31,79 +30,165 @@ func main() {
r := chi.NewRouter()
r.Use(middleware.Logger)
r.Post("/", func(w http.ResponseWriter, r *http.Request) {
- if r.Header.Get("Authorization") != "Basic "+ os.Getenv("CRON_SECRET") {
+ if r.Header.Get("Authorization") != "Basic "+os.Getenv("CRON_SECRET") {
http.Error(w, "Unauthorized", 401)
return
- }
- region := os.Getenv("FLY_REGION")
+ }
+ i, err := strconv.Atoi(r.Header.Get("X-CloudTasks-TaskRetryCount"))
+ if err != nil {
+ http.Error(w, "Something went whont", 400)
+ return
+ }
+ // If something went wrong we only try it twice
+ if i > 1 {
+ w.WriteHeader(http.StatusOK)
+ w.Write([]byte("Ok"))
+ return
+ }
+
if r.Body == nil {
http.Error(w, "Please send a request body", 400)
return
}
var u InputData
+ // region := os.Getenv("FLY_REGION")
- err := json.NewDecoder(r.Body).Decode(&u)
+ err = json.NewDecoder(r.Body).Decode(&u)
- fmt.Printf("Start checker for %+v", u)
+ fmt.Printf("πŸš€ Start checker for %+v \n", u)
if err != nil {
- http.Error(w, err.Error(), 400)
+ w.Write([]byte("Ok"))
+ w.WriteHeader(200)
return
}
- request, error := http.NewRequest(u.Method, u.Url, bytes.NewReader([]byte(u.Body)))
- // Setting headers
- for _, header := range u.Headers {
- fmt.Printf("%+v", header)
- if header.Key != "" && header.Value != "" {
- request.Header.Set(header.Key, header.Value)
+ client := &http.Client{}
+ defer client.CloseIdleConnections()
+
+ response, error := ping(client, u)
+
+ if error != nil {
+ sendToTinybird(response)
+ if u.Status == "active" {
+ // updateStatus(UpdateData{
+ // MonitorId: u.MonitorId,
+ // Status: "error",
+ // Message: error.Error(),
+ // Region: region,
+ // })
}
+ w.Write([]byte("Ok"))
+ w.WriteHeader(200)
+ return
}
- if error != nil {
- fmt.Println(error)
+ sendToTinybird(response)
+
+ if response.StatusCode < 200 || response.StatusCode >= 300 {
+ // If the status code is not within the 200 range, we update the status to error
+ // updateStatus(UpdateData{
+ // MonitorId: u.MonitorId,
+ // Status: "error",
+ // StatusCode: response.StatusCode,
+ // Region: region,
+ // })
+ }
+ if u.Status == "error" {
+ // If the status was error, we update it to active
+ // updateStatus(UpdateData{
+ // MonitorId: u.MonitorId,
+ // Status: "active",
+ // Region: region,
+ // StatusCode: response.StatusCode,
+ // })
+ }
+
+ fmt.Printf("⏱️ End checker for %+v with latency %+d and statusCode %+d", u, response.Latency, response.StatusCode)
+ w.Write([]byte("Ok"))
+ w.WriteHeader(200)
+ return
+ })
+ // That's the new checker sending to the correct ingest endpoint
+ r.Post("/checker", func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "Basic "+os.Getenv("CRON_SECRET") {
+ http.Error(w, "Unauthorized", 401)
+ return
+ }
+ i, err := strconv.Atoi(r.Header.Get("X-CloudTasks-TaskRetryCount"))
+ if err != nil {
+ http.Error(w, "Something went whont", 400)
+ return
+ }
+ // If something went wrong we only try it twice
+ if i > 1 {
+ w.WriteHeader(http.StatusOK)
+ w.Write([]byte("Ok"))
+ return
+ }
+
+ if r.Body == nil {
+ http.Error(w, "Please send a request body", 400)
+ return
+ }
+ var u InputData
+ // region := os.Getenv("FLY_REGION")
+
+ err = json.NewDecoder(r.Body).Decode(&u)
+
+ fmt.Printf("πŸš€ Start checker for %+v \n", u)
+
+ if err != nil {
+ w.Write([]byte("Ok"))
+ w.WriteHeader(200)
+ return
}
client := &http.Client{}
- start := time.Now().UTC().UnixMilli()
- response, error := client.Do(request)
- end := time.Now().UTC().UnixMilli()
+ defer client.CloseIdleConnections()
+
+ response, error := ping(client, u)
- // Retry if error
if error != nil {
- response, error = client.Do(request)
- end = time.Now().UTC().UnixMilli()
+ sendToTinybirdNew(response)
+ if u.Status == "active" {
+ // updateStatus(UpdateData{
+ // MonitorId: u.MonitorId,
+ // Status: "error",
+ // Message: error.Error(),
+ // Region: region,
+ // })
+ }
+ w.Write([]byte("Ok"))
+ w.WriteHeader(200)
+ return
}
- latency := end - start
- fmt.Println("πŸš€ Checked url: %v with latency %v in region %v ", u.Url, latency, region)
- fmt.Printf("Response %+v for %+v", response, u)
- if error != nil {
- tiny((PingData{
- Latency: (latency),
- MonitorId: u.MonitorId,
- Region: region,
- WorkspaceId: u.WorkspaceId,
- Timestamp: time.Now().UTC().UnixMilli(),
- Url: u.Url,
- Message: error.Error(),
- }))
- } else {
- tiny((PingData{
- Latency: (latency),
- MonitorId: u.MonitorId,
- Region: region,
- WorkspaceId: u.WorkspaceId,
- StatusCode: int16(response.StatusCode),
- Timestamp: time.Now().UTC().UnixMilli(),
- Url: u.Url,
- }))
- }
-
- fmt.Printf("End checker for %+v", u)
+ sendToTinybirdNew(response)
+
+ if response.StatusCode < 200 || response.StatusCode >= 300 {
+ // If the status code is not within the 200 range, we update the status to error
+ // updateStatus(UpdateData{
+ // MonitorId: u.MonitorId,
+ // Status: "error",
+ // StatusCode: response.StatusCode,
+ // Region: region,
+ // })
+ }
+ if u.Status == "error" {
+ // If the status was error, we update it to active
+ // updateStatus(UpdateData{
+ // MonitorId: u.MonitorId,
+ // Status: "active",
+ // Region: region,
+ // StatusCode: response.StatusCode,
+ // })
+ }
+ fmt.Printf("⏱️ End checker for %+v with latency %+d and statusCode %+d", u, response.Latency, response.StatusCode)
w.Write([]byte("Ok"))
w.WriteHeader(200)
+ return
})
r.Get("/ping", func(w http.ResponseWriter, r *http.Request) {
@@ -116,8 +201,9 @@ func main() {
}
w.Header().Set("Content-Type", "application/json")
- w.WriteHeader(http.StatusCreated)
+ w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode(data)
+ return
})
http.ListenAndServe(":8080", r)
diff --git a/apps/checker/ping.go b/apps/checker/ping.go
index 0b4902dc9..8c55c7769 100644
--- a/apps/checker/ping.go
+++ b/apps/checker/ping.go
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"net/http"
+ "net/url"
"os"
"time"
)
@@ -14,17 +15,17 @@ type PingData struct {
WorkspaceId string `json:"workspaceId"`
MonitorId string `json:"monitorId"`
Timestamp int64 `json:"timestamp"`
- StatusCode int16 `json:"statusCode"`
+ StatusCode int `json:"statusCode,omitempty"`
Latency int64 `json:"latency"`
CronTimestamp int64 `json:"cronTimestamp"`
Url string `json:"url"`
Region string `json:"region"`
- Message string `json:"message"`
+ Message string `json:"message,omitempty"`
}
-func tiny(pingData PingData) {
- url := "https://api.tinybird.co/v0/events?name=golang_ping_response__v1"
- fmt.Println("URL:>", url)
+func sendToTinybird(pingData PingData) {
+ url := "https://api.tinybird.co/v0/events?name=golang_ping_response__v3"
+ fmt.Printf("πŸ“ˆ Sending data to Tinybird for %+v \n", pingData)
bearer := "Bearer " + os.Getenv("TINYBIRD_TOKEN")
payloadBuf := new(bytes.Buffer)
json.NewEncoder(payloadBuf).Encode(pingData)
@@ -33,13 +34,84 @@ func tiny(pingData PingData) {
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: time.Second * 10}
- resp, err := client.Do(req)
+ _, err = client.Do(req)
if err != nil {
fmt.Println(err)
+ panic(err)
+ }
+ // Should we add a retry mechanism here?
+
+}
+
+func sendToTinybirdNew(pingData PingData) {
+ url := "https://api.tinybird.co/v0/events?name=ping_response__v5"
+ fmt.Printf("πŸ“ˆ Sending data to Tinybird for %+v \n", pingData)
+ bearer := "Bearer " + os.Getenv("TINYBIRD_TOKEN")
+ payloadBuf := new(bytes.Buffer)
+ json.NewEncoder(payloadBuf).Encode(pingData)
+ req, err := http.NewRequest("POST", url, payloadBuf)
+ req.Header.Set("Authorization", bearer)
+ req.Header.Set("Content-Type", "application/json")
+
+ client := &http.Client{Timeout: time.Second * 10}
+ _, err = client.Do(req)
+ if err != nil {
+ fmt.Println(err)
+ panic(err)
+ }
+ // Should we add a retry mechanism here?
+
+}
+
+func ping(client *http.Client, inputData InputData) (PingData, error) {
+
+ region := os.Getenv("FLY_REGION")
+ request, err := http.NewRequest(inputData.Method, inputData.Url, bytes.NewReader([]byte(inputData.Body)))
+
+ if err != nil {
+ return PingData{}, fmt.Errorf("Unable to create request: %w", err)
+ }
+ request.Header.Set("User-Agent", "OpenStatus/1.0")
+
+ // Setting headers
+ for _, header := range inputData.Headers {
+ if header.Key != "" && header.Value != "" {
+ request.Header.Set(header.Key, header.Value)
+ }
+ }
+
+ start := time.Now()
+ response, err := client.Do(request)
+ latency := time.Since(start).Milliseconds()
+ defer response.Body.Close()
+
+ _, err = io.ReadAll(response.Body)
+ if err != nil {
+ if urlErr, ok := err.(*url.Error); ok {
+ if urlErr.Timeout() {
+ return PingData{
+ Latency: latency,
+ MonitorId: inputData.MonitorId,
+ Region: region,
+ WorkspaceId: inputData.WorkspaceId,
+ Timestamp: time.Now().UTC().UnixMilli(),
+ Url: inputData.Url,
+ Message: fmt.Sprintf("Timeout after %d ms", latency),
+ }, nil
+ }
+ }
+ return PingData{}, fmt.Errorf("Error while reading body from %s: %w", inputData.Url, err)
}
- defer resp.Body.Close()
- body, _ := io.ReadAll(resp.Body)
- fmt.Println(string(body))
+ return PingData{
+ Latency: latency,
+ StatusCode: response.StatusCode,
+ MonitorId: inputData.MonitorId,
+ Region: region,
+ WorkspaceId: inputData.WorkspaceId,
+ Timestamp: time.Now().UTC().UnixMilli(),
+ CronTimestamp: inputData.CronTimestamp,
+ Url: inputData.Url,
+ }, nil
}
diff --git a/apps/checker/update.go b/apps/checker/update.go
new file mode 100644
index 000000000..77696e2d3
--- /dev/null
+++ b/apps/checker/update.go
@@ -0,0 +1,36 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "os"
+ "time"
+)
+
+type UpdateData struct {
+ MonitorId string `json:"monitorId"`
+ Status string `json:"status"`
+ Message string `json:"message,omitempty"`
+ StatusCode int `json:"statusCode,omitempty"`
+ Region string `json:"region"`
+}
+
+func updateStatus(updateData UpdateData) {
+ url := "https://openstatus-api.fly.dev/updateStatus"
+ fmt.Println("URL:>", url)
+ bearer := "Bearer " + os.Getenv("CRON_SECRET")
+ payloadBuf := new(bytes.Buffer)
+ json.NewEncoder(payloadBuf).Encode(updateData)
+ req, err := http.NewRequest("POST", url, payloadBuf)
+ req.Header.Set("Authorization", bearer)
+ req.Header.Set("Content-Type", "application/json")
+
+ client := &http.Client{Timeout: time.Second * 10}
+ _, err = client.Do(req)
+ if err != nil {
+ panic(err)
+ }
+ // Should we add a retry mechanism here?
+}
diff --git a/apps/server/src/checker/alerting.ts b/apps/server/src/checker/alerting.ts
index b103465f5..05abc42c3 100644
--- a/apps/server/src/checker/alerting.ts
+++ b/apps/server/src/checker/alerting.ts
@@ -1,12 +1,14 @@
import { db, eq, schema } from "@openstatus/db";
-import type { MonitorRegion, MonitorStatus } from "@openstatus/db/src/schema";
+import type {
+ MonitorFlyRegion,
+ MonitorStatus,
+} from "@openstatus/db/src/schema";
import {
selectMonitorSchema,
selectNotificationSchema,
} from "@openstatus/db/src/schema";
import { flyRegionsDict } from "@openstatus/utils";
-import { env } from "../env";
import { checkerAudit } from "../utils/audit-log";
import { providerToFunction } from "./utils";
@@ -58,11 +60,12 @@ export const triggerAlerting = async ({
export const upsertMonitorStatus = async ({
monitorId,
status,
+ region,
}: {
monitorId: string;
status: MonitorStatus;
+ region: MonitorFlyRegion;
}) => {
- const region = env.FLY_REGION as MonitorRegion;
await db
.insert(schema.monitorStatusTable)
.values({ status, region, monitorId: Number(monitorId) })
diff --git a/apps/server/src/checker/index.ts b/apps/server/src/checker/index.ts
index ee907c520..7f821d1c2 100644
--- a/apps/server/src/checker/index.ts
+++ b/apps/server/src/checker/index.ts
@@ -1,6 +1,11 @@
import { Hono } from "hono";
+import { z } from "zod";
+
+import { flyRegions } from "@openstatus/db/src/schema/monitors/constants";
import { env } from "../env";
+import { checkerAudit } from "../utils/audit-log";
+import { upsertMonitorStatus } from "./alerting";
import { checkerRetryPolicy } from "./checker";
import { payloadSchema } from "./schema";
import type { Payload } from "./schema";
@@ -113,3 +118,67 @@ checkerRoute.post("/checkerV2", async (c) => {
return c.text("Internal Server Error", 500);
}
});
+
+checkerRoute.post("/statusChange", async (c) => {
+ const auth = c.req.header("Authorization");
+ if (auth !== `Basic ${env.CRON_SECRET}`) {
+ console.error("Unauthorized");
+ return c.text("Unauthorized", 401);
+ }
+
+ const json = await c.req.json();
+ const schema = z.object({
+ monitorId: z.string(),
+ status: z.enum(["active", "error"]), // that's the new status
+ message: z.string().optional(),
+ statusCode: z.number().optional(),
+ region: z.enum(flyRegions),
+ });
+
+ const result = schema.safeParse(json);
+ if (!result.success) {
+ // console.error(result.error);
+ return c.text("Unprocessable Entity", 422);
+ }
+ const { monitorId, status, message, region, statusCode } = result.data;
+
+ console.log(`πŸ“ update monitor status ${JSON.stringify(result.data)}`);
+
+ switch (status) {
+ case "active":
+ await upsertMonitorStatus({
+ monitorId: monitorId,
+ status: "active",
+ region: region,
+ });
+ if (!statusCode) {
+ return;
+ }
+ await checkerAudit.publishAuditLog({
+ id: `monitor:${monitorId}`,
+ action: "monitor.recovered",
+ targets: [{ id: monitorId, type: "monitor" }],
+ metadata: { region: region, statusCode: statusCode },
+ });
+ break;
+ case "error":
+ await upsertMonitorStatus({
+ monitorId: monitorId,
+ status: "error",
+ region: region,
+ });
+ // ALPHA
+ await checkerAudit.publishAuditLog({
+ id: `monitor:${monitorId}`,
+ action: "monitor.failed",
+ targets: [{ id: monitorId, type: "monitor" }],
+ metadata: {
+ region: region,
+ statusCode: statusCode,
+ message,
+ },
+ });
+ break;
+ }
+ return c.text("Ok", 200);
+});
diff --git a/packages/db/src/schema/monitors/validation.ts b/packages/db/src/schema/monitors/validation.ts
index 127242e28..954bffbea 100644
--- a/packages/db/src/schema/monitors/validation.ts
+++ b/packages/db/src/schema/monitors/validation.ts
@@ -2,6 +2,7 @@ import { createInsertSchema, createSelectSchema } from "drizzle-zod";
import { z } from "zod";
import {
+ flyRegions,
monitorJobTypes,
monitorMethods,
monitorPeriodicity,
@@ -15,6 +16,7 @@ export const monitorMethodsSchema = z.enum(monitorMethods);
export const monitorStatusSchema = z.enum(monitorStatus);
export const monitorRegionSchema = z.enum(monitorRegions);
export const monitorJobTypesSchema = z.enum(monitorJobTypes);
+export const monitorFlyRegionSchema = z.enum(flyRegions);
// TODO: shared function
function stringToArrayProcess<T>(string: T) {}
@@ -79,4 +81,5 @@ export type MonitorStatus = z.infer<typeof monitorStatusSchema>;
export type MonitorPeriodicity = z.infer<typeof monitorPeriodicitySchema>;
export type MonitorMethod = z.infer<typeof monitorMethodsSchema>;
export type MonitorRegion = z.infer<typeof monitorRegionSchema>;
+export type MonitorFlyRegion = z.infer<typeof monitorFlyRegionSchema>;
export type MonitorJobType = z.infer<typeof monitorJobTypesSchema>;